class Program
{
static void Main(string[] args)
{
LogFileGenerator gen = new LogFileGenerator();
gen.GenerateAndSave("D:\\logs", new DateTime(2017, 9, 13), new DateTime(2017, 9, 14));
LogStatistics logstat= new LogStatistics("D:\\logs");
logstat.UserStatistics();
logstat.DomainStatistics();
logstat.DateStatistics();
logstat.SaveUserStatReport("D:\\logs\\user_report.txt");
logstat.SaveDomainStatReport("D:\\logs\\domain_report.txt");
logstat.SaveDateStatReport("D:\\logs\\date_report.txt");
Console.ReadKey(true);
}
}
[Serializable]
class LogItem
{
public DateTime date;
public int userIP;
public int domainIP;
public string domainName = "";
public ulong trafficSize;
public LogItem() { }
public LogItem(DateTime date, int userIP, int domainIP, string domainName, ulong trafficSize)
{
this.date = date;
this.userIP = userIP;
this.domainIP = domainIP;
this.trafficSize = trafficSize;
this.domainName = domainName;
}
}
class LogFileGenerator
{
List<string> letters = new List<string>(){"a","b", "c","d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s","t", "u", "v", "w", "x", "y", "z" };
List<string> domains = new List<string>() { ".com", ".ru", ".su" };
public void GenerateAndSave(string pathToSave, DateTime startDate, DateTime endDate)
{
Random rand = new Random();
List<List<LogItem>> logPackage = new List<List<LogItem>>();
int max = endDate.Day - startDate.Day;
for (int i = 0; i <= max; i++)
{
List<LogItem> list = new List<LogItem>();
for (int k = 0; k < rand.Next(2, 5); k++)
{
string domainname = "";
for (int j = 0; j < rand.Next(4, 10); j++)
{
domainname += letters[rand.Next(0, letters.Count - 1)];
}
domainname += domains[rand.Next(0, domains.Count - 1)];
list.Add(new LogItem(startDate.AddDays(i), rand.Next(12345, 12350), rand.Next(45678, 45680), domainname, (ulong)rand.Next(1, 10)));
}
using (Stream st = new FileStream(pathToSave+"\"+startDate.AddDays(i).ToShortDateString()+ ".log", FileMode.Create))
{
IFormatter f = new BinaryFormatter();
f.Serialize(st, list);
}
foreach(var l in list)
{
List<string> IP = new List<string>();
int mask = 255;
for (int j = 0; j < 4; j++)
{
IP.Add(((l.userIP >> j * 8) & mask).ToString());
}
IP.Reverse();
string uip = String.Join(".", IP);
IP.Clear();
for (int j = 0; j < 4; j++)
{
IP.Add(((l.domainIP >> j * 8) & mask).ToString());
}
IP.Reverse();
string dip = String.Join(".", IP);
Console.WriteLine(l.date.ToShortDateString() +" " + uip + " " + dip + " " + l.trafficSize);
}
}
}
}
class LogStatistics
{
private List<Dictionary<int, ulong>> user = new List<Dictionary<int, ulong>>();
private List<Dictionary<int, ulong>> domain = new List<Dictionary<int, ulong>>();
private List<Dictionary<DateTime, ulong>> date= new List<Dictionary<DateTime, ulong>>();
public Dictionary<int, ulong> user_stat;
public Dictionary<int, ulong> domain_stat;
public Dictionary<DateTime, ulong> date_stat;
List<string> logFiles;
public LogStatistics(string logDir)
{
user_stat = new Dictionary<int, ulong>();
domain_stat = new Dictionary<int, ulong>();
date_stat = new Dictionary<DateTime, ulong>();
logFiles = new List<string>();
logFiles = System.IO.Directory.GetFiles(logDir,"*.log").ToList();
}
public void UserStatistics()
{
var events = new List<ManualResetEvent>();
ThreadPool.SetMaxThreads(4, 4);
foreach(var file in logFiles)
{
var resetEvent = new ManualResetEvent(false);
ThreadPool.QueueUserWorkItem(arg =>
{
UserStatisticsWorker(file);
resetEvent.Set();
});
events.Add(resetEvent);
}
WaitHandle.WaitAll(events.ToArray());
Console.WriteLine("User stat done");
foreach (var piece in user)
{
foreach (var item in piece)
{
if (user_stat.ContainsKey(item.Key))
{
user_stat[item.Key] += item.Value;
}
else
{
user_stat.Add(item.Key, item.Value);
}
}
}
}
public void DomainStatistics()
{
var events = new List<ManualResetEvent>();
ThreadPool.SetMaxThreads(4, 4);
ThreadPool.SetMinThreads(4, 4);
foreach (var file in logFiles)
{
var resetEvent = new ManualResetEvent(false);
ThreadPool.QueueUserWorkItem(arg =>
{
DomainStatisticsWorker(file);
resetEvent.Set();
});
events.Add(resetEvent);
}
WaitHandle.WaitAll(events.ToArray());
Console.WriteLine("Domain stat done");
foreach (var piece in domain)
{
foreach (var item in piece)
{
if (domain_stat.ContainsKey(item.Key))
{
domain_stat[item.Key] += item.Value;
}
else
{
domain_stat.Add(item.Key, item.Value);
}
}
}
}
public void DateStatistics()
{
var events = new List<ManualResetEvent>();
ThreadPool.SetMaxThreads(4, 4);
foreach(var file in logFiles)
{
var resetEvent = new ManualResetEvent(false);
ThreadPool.QueueUserWorkItem(arg =>
{
DateStatisticsWorker(file);
resetEvent.Set();
});
events.Add(resetEvent);
}
WaitHandle.WaitAll(events.ToArray());
Console.WriteLine("Date stat done");
foreach (var piece in date)
{
foreach (var item in piece)
{
if (date_stat.ContainsKey(item.Key))
{
date_stat[item.Key] += item.Value;
}
else
{
date_stat.Add(item.Key, item.Value);
}
}
}
}
public void SaveUserStatReport(string dirPath)
{
string output = "";
foreach (var item in user_stat)
{
List<string> IP = new List<string>();
int mask = 255;
for (int i = 0; i < 4; i++)
{
IP.Add(((item.Key >> i*8) & mask).ToString());
}
IP.Reverse();
output += String.Join(".",IP);
output += ": " + item.Value + "byte\r\n";
}
File.WriteAllText(dirPath, output);
}
public void SaveDomainStatReport(string dirPath)
{
string output = "";
foreach (var item in domain_stat)
{
List<string> IP = new List<string>();
int mask = 255;
for (int i = 0; i < 4; i++)
{
IP.Add(((item.Key >> i * 8) & mask).ToString());
}
IP.Reverse();
output += String.Join(".", IP);
output += ": " + item.Value + "byte\r\n";
}
File.WriteAllText(dirPath, output);
}
public void SaveDateStatReport(string dirPath)
{
string output = "";
foreach (var item in date_stat)
{
output += item.Key.ToShortDateString()+": " + item.Value + "byte\r\n";
}
File.WriteAllText(dirPath, output);
}
private void UserStatisticsWorker(object state)
{
Console.WriteLine("user");
List<LogItem> log = new List<LogItem>();
string logPath = (string)state;
using (Stream st = new FileStream(logPath, FileMode.Open, FileAccess.Read))
{
IFormatter f = new BinaryFormatter();
log = (List<LogItem>)f.Deserialize(st);
}
Dictionary<int, ulong> stat = new Dictionary<int, ulong>();
foreach (var item in log)
{
if(stat.ContainsKey(item.userIP))
{
stat[item.userIP] += item.trafficSize;
}
else
{
stat.Add(item.userIP, item.trafficSize);
}
}
user.Add(stat);
}
private void DomainStatisticsWorker(object state)
{
Console.WriteLine("domain");
List<LogItem> log = new List<LogItem>();
string logPath = (string)state;
using (Stream st = new FileStream(logPath, FileMode.Open, FileAccess.Read))
{
IFormatter f = new BinaryFormatter();
log = (List<LogItem>)f.Deserialize(st);
}
Dictionary<int, ulong> stat = new Dictionary<int, ulong>();
foreach (var item in log)
{
if (stat.ContainsKey(item.domainIP))
{
stat[item.domainIP] += item.trafficSize;
}
else
{
stat.Add(item.domainIP, item.trafficSize);
}
}
domain.Add(stat);
}
private void DateStatisticsWorker(object state)
{
List<LogItem> log = new List<LogItem>();
string logPath = (string)state;
using (Stream st = new FileStream(logPath, FileMode.Open, FileAccess.Read))
{
IFormatter f = new BinaryFormatter();
log = (List<LogItem>)f.Deserialize(st);
}
Dictionary<DateTime, ulong> dic = new Dictionary<DateTime, ulong>();
foreach (var item in log)
{
if (dic.ContainsKey(item.date))
{
dic[item.date] += item.trafficSize;
}
else
{
dic.Add(item.date, item.trafficSize);
}
}
date.Add(dic);
}
}
var events = new List<ManualResetEvent>(); // создаем список событий
ThreadPool.SetMaxThreads(4, 4); // задает количество запросов к пулу потоков
// цикл перебора файлов
foreach (var file in logFiles)
{
// создание задачи
var resetEvent = new ManualResetEvent(false); // создаем новое событие
ThreadPool.QueueUserWorkItem( // помещает в очередь на выполнение
arg => // получаем объект. в данном случае будет задача
{
UserStatisticsWorker(file); // вызываем метод для подсчета статистики по пользователя из файла
resetEvent.Set(); // устанавливает состояние, что поток занят
});
events.Add(resetEvent); // добавляем в список потоков
}
WaitHandle.WaitAll(events.ToArray()); // ждем завершения все потоков
Если Вы уже зарегистрированы на Портале - войдите в систему, если Вы еще не регистрировались - пройдите простую процедуру регистрации.