using NLog; using NLog.Config; using NLog.Targets; using System; using System.Net; using System.Net.Http; using DataClient; using DataClient.Struct; using System.Linq; using System.Threading.Tasks; using System.Net.Http.Headers; using ApiServer.ApiStruct; using Newtonsoft.Json; using System.Text; using System.IO; using System.Collections.Generic; using System.Diagnostics; using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; using System.Net.Security; namespace ClientCollector { class Program { static Logger log = LogManager.GetCurrentClassLogger(); static void Main(string[] args) { LogConf(); log.Info("Start Client Collector."); Task taskPasp = null; var workPasp = false; Task taskData = null; var workData = false; foreach(var arg in args) { if(int.TryParse(arg, out int flag)) switch (flag) { case 1: workPasp = true; break; case 2: workData = true; break; } } while (true) { if (taskPasp == null || taskPasp.IsCompleted) { log.Info(workPasp ? "Start Pasport Task." : "Skip Pasport Task."); taskPasp = workPasp ? WorkPasport() : Task.Delay(int.MaxValue); } if (taskData == null || taskData.IsCompleted) { log.Info(workData ? "Start Data Task." : "Skip Data Task."); taskData = workData ? WorkData() : Task.Delay(int.MaxValue); } log.Info("Wait tasks."); Task.WaitAny(new Task[] { taskPasp, taskData }); } } static async Task WorkPasport() { await Task.Delay(1000); var IpSTP = "10.10.45.152"; var PortSTP = 1070; var nameCurrDate = "pasport"; var currDate = GetCurrData(nameCurrDate); if (!currDate.HasValue) currDate = new DateTime(2001, 02, 01); NETClient netClient = new NETClient(IpSTP, PortSTP); var flagcycle = false; while (currDate.Value < DateTime.Now.AddDays(-1)) { try { var currDir = currDate.Value.Year.ToString("D4") + '/' + currDate.Value.Month.ToString("D2") + '/' + currDate.Value.Day.ToString("D2"); log.Info("Get pasports from: " + currDir); while (!netClient.Connected() || flagcycle) { flagcycle = false; if (!netClient.ReConnect()) { netClient.Close(); netClient = new NETClient(IpSTP, PortSTP); log.Warn("Can't connect to STP."); await Task.Delay(10000); } } var currPasports = netClient.Full_Dir_Browse(currDir); netClient.Close(); if (currPasports == null) currPasports = Array.Empty(); foreach (var paspDir in currPasports) { do { try { log.Info("Get pasport: " + paspDir); while (!netClient.Connected() || flagcycle) { flagcycle = false; if (!netClient.ReConnect()) { log.Warn("Can't connect to STP."); netClient.Close(); netClient = new NETClient(IpSTP, PortSTP); await Task.Delay(10000); } } var pasp = netClient.Full_Pasp_Download(paspDir); netClient.Close(); if (pasp == null || !pasp.HasData) { log.Warn("Can't get pasport."); netClient.Close(); netClient = new NETClient(IpSTP, PortSTP); flagcycle = true; await Task.Delay(10000); continue; } while (!SendPasport(pasp)) { log.Warn("Can't send pasp to API."); await Task.Delay(10000); } break; } catch (Exception e) { log.Warn(e.Message); netClient.Close(); flagcycle = true; await Task.Delay(5000); } } while (true); } log.Info("End day: " + currDir); SaveCurrData(nameCurrDate, currDate.Value); currDate = currDate.Value.AddDays(1); await Task.Delay(2000); } catch (Exception e) { log.Warn(e.Message); flagcycle = true; netClient.Close(); netClient = new NETClient(IpSTP, PortSTP); await Task.Delay(1000 * 60 * 5); } } do { log.Info("Wait next day."); await Task.Delay(1000 * 60 * 60); } while (!(currDate.Value < DateTime.Now.AddDays(-1))); currDate.Value.AddDays(-15); SaveCurrData(nameCurrDate, currDate.Value); } static async Task WorkData() { await Task.Delay(1000); var IpSTP = "10.10.45.152"; var PortSTP = 1070; var mainDir = Path.Combine(Directory.GetCurrentDirectory(), "archive"); var nameCurrDate = "data"; var currDate = GetCurrData(nameCurrDate); if (!currDate.HasValue) currDate = new DateTime(2001, 02, 01); while (currDate.Value < DateTime.Now.AddDays(-1)) { try { while(DateTime.Now.Hour < 8) { log.Info("Await 8:00."); await Task.Delay(15 * 1000 * 60); } log.Info("Disconnect archive."); while (DisconnectNFS(mainDir)) await Task.Delay(1000); log.Info("Connect to archive."); while (!ConnectNFS(mainDir)) await Task.Delay(1000); var subDir = Path.Combine(mainDir, "data", currDate.Value.Year.ToString("D4"), currDate.Value.Month.ToString("D2"), currDate.Value.Day.ToString("D2")); log.Info("Check dir: " + subDir); if (Directory.Exists(Path.Combine(mainDir, "data"))) { if (Directory.Exists(subDir)) { var listFiles = Directory.GetFiles(subDir); foreach (var fileDir in listFiles) { log.Info("Get file: " + fileDir); var fileStruct = File.ReadAllBytes(fileDir); log.Info("Send file to API: " + fileDir); while (!SendData(currDate.Value, Path.GetFileName(fileDir), fileStruct)) { log.Warn("Can't send data to API."); await Task.Delay(10000); } } } else { var listVDP = new List(); for (var i = 0; i < 50; i++) listVDP.Add(i); for (var i = 90; i < 95; i++) listVDP.Add(i); foreach (var vdp in listVDP) { var pathTmp = Path.Combine(Directory.GetCurrentDirectory(), "tmpData"); if (Directory.Exists(pathTmp)) Directory.Delete(pathTmp, true); Directory.CreateDirectory(pathTmp); for (var numFile = 0; numFile < 16; numFile++) { await Task.Delay(2000); var name = currDate.Value.ToString("yyyyMMdd") + "." + vdp.ToString("D2") + numFile.ToString("X1"); try { var nc = new NETClient(IpSTP, PortSTP); while (!nc.Connected()) if (!nc.ReConnect()) { log.Warn("Can't connect to STP."); await Task.Delay(5000); } log.Info("Try download " + name); var array = nc.Full_Download_NH(currDate.Value, vdp, numFile); nc.Close(); if (array == null || array.Length == 0) { log.Info("File " + name + " not exist."); continue; } File.WriteAllBytes(Path.Combine(pathTmp, name), array); } catch (Exception e) { log.Warn(e.Message); log.Warn("Can't download " + name); } } { var files = Directory.GetFiles(pathTmp); if (files.Length > 0) { var targzFile = vdp.ToString("D2") + ".tar.gz"; using (var outStream = File.Create(Path.Combine(pathTmp, targzFile))) using (var gzoStream = new GZipOutputStream(outStream)) using (var tarArchive = TarArchive.CreateOutputTarArchive(gzoStream)) foreach (var file in files) { tarArchive.RootPath = Path.GetDirectoryName(file); var tarEntry = TarEntry.CreateEntryFromFile(file); tarEntry.Name = Path.GetFileName(file); tarArchive.WriteEntry(tarEntry, true); } var fileStruct = File.ReadAllBytes(Path.Combine(pathTmp, targzFile)); while (!SendData(currDate.Value, Path.GetFileName(targzFile), fileStruct)) { log.Warn("Can't send data to API."); await Task.Delay(10000); } } Directory.Delete(pathTmp, true); } } } log.Info("End day: " + subDir); SaveCurrData(nameCurrDate, currDate.Value); currDate = currDate.Value.AddDays(1); } else { log.Warn("Archive not exist."); } } catch (Exception e) { log.Warn(e.Message); } } do { log.Info("Wait next day."); await Task.Delay(1000 * 60 * 60); } while (!(currDate.Value < DateTime.Now.AddDays(-1))); currDate.Value.AddDays(-15); SaveCurrData(nameCurrDate, currDate.Value); } static bool ConnectNFS(string dir) { var dirCreated = false; while (!dirCreated) { try { dirCreated = Directory.Exists(Path.Combine(dir)); if (!dirCreated) Directory.CreateDirectory(dir); } catch (Exception e) { log.Warn(e.Message); log.Warn("Can't create directory"); Task.Delay(60000).Wait(); } } try { var procC = new Process { StartInfo = new ProcessStartInfo { FileName = "/usr/bin/mount", Arguments = "-t nfs 10.10.45.236:/archiv " + dir, UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, }, }; if (!procC.Start()) { log.Warn("Can't start mount command."); Task.Delay(60000).Wait(); } var answer = procC.StandardError.ReadToEnd() + procC.StandardOutput.ReadToEnd(); var connected = string.IsNullOrEmpty(answer); if (connected) { log.Info("Connect Success"); return true; } log.Info(answer); return false; } catch (Exception e) { log.Warn(e.Message); log.Warn("Can't connect to NFS."); Task.Delay(60000).Wait(); return false; } } static bool DisconnectNFS(string dir) { var dirCreated = false; while (!dirCreated) { try { dirCreated = Directory.Exists(Path.Combine(dir)); if (!dirCreated) Directory.CreateDirectory(dir); } catch (Exception e) { log.Warn(e.Message); log.Warn("Can't create directory"); Task.Delay(60000).Wait(); } } try { var procD = new Process { StartInfo = new ProcessStartInfo { FileName = "/usr/bin/umount", Arguments = dir, UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, } }; if (!procD.Start()) { log.Warn("Can't start umount command."); Task.Delay(60000).Wait(); } var answer = procD.StandardError.ReadToEnd() + procD.StandardOutput.ReadToEnd(); log.Info(string.IsNullOrEmpty(answer) ? "NFS disconnected." : answer); return string.IsNullOrEmpty(answer); } catch (Exception e) { log.Warn(e.Message); return false; } } static HttpWebRequest GetRequest(string path, bool useProxy = true) { HttpWebRequest clientReq = (HttpWebRequest)WebRequest.Create(path); clientReq.ServerCertificateValidationCallback = (sender, cert, chain, error) => { return true; }; clientReq.Method = "POST"; clientReq.ContentType = "application/json; charset=utf-8"; if (useProxy) { clientReq.Proxy = new WebProxy("194.226.128.245", 3128) { BypassProxyOnLocal = false, Credentials = new NetworkCredential("user4", "user4") }; } return clientReq; } static bool SendPasport(Pasport pasp) { try { { log.Info("Check pasport on remote API server."); var req = JsonConvert.SerializeObject( new PasportCheckApi() { Status = true, DateAndTime = pasp.dEnd.Value, Name = string.IsNullOrEmpty(pasp.nplav) ? pasp.numVDP.Value.ToString("D2") : pasp.nplav }); var reqArr = Encoding.UTF8.GetBytes(req); HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/pasport/check"); clientReq.ContentLength = reqArr.Length; using (var stream = clientReq.GetRequestStream()) { stream.Write(reqArr, 0, reqArr.Length); } string response = ""; using (var clientRes = (HttpWebResponse)clientReq.GetResponse()) { using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8)) { response = readStream.ReadToEnd(); readStream.Close(); clientRes.Close(); } } var rep = JsonConvert.DeserializeObject(response); if (!rep.Status) return false; ulong paspSum = 0; var paspArr = pasp.PaspByte; foreach (var b in paspArr) paspSum += b; if (rep.Exist && rep.PaspSum == paspSum) return true; if (!rep.Exist) log.Info("API: Pasport not exist."); if (rep.Exist && rep.PaspSum != paspSum) log.Info("API: Wrong size pasport."); } { log.Info("Send pasport to remote API server."); var req = JsonConvert.SerializeObject( new PasportCreateApi() { Status = true, Pasp = pasp }); var reqArr = Encoding.UTF8.GetBytes(req); HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/pasport/create"); clientReq.ContentLength = reqArr.Length; using (var stream = clientReq.GetRequestStream()) { stream.Write(reqArr, 0, reqArr.Length); } string response = ""; using (var clientRes = (HttpWebResponse)clientReq.GetResponse()) { using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8)) { response = readStream.ReadToEnd(); readStream.Close(); clientRes.Close(); } } var rep = JsonConvert.DeserializeObject(response); if (!rep.Status) return false; } return true; } catch (Exception e) { log.Warn(e.Message); return false; } } static bool SendData(DateTime dateAndTime, string fileName, byte[] fileStruct) { try { { log.Info("Check data on remote API server."); var req = JsonConvert.SerializeObject( new DataCheckApi() { Status = true, DateAndTime = dateAndTime, Name = fileName }); var reqArr = Encoding.UTF8.GetBytes(req); HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/data/check"); clientReq.ContentLength = reqArr.Length; using (var stream = clientReq.GetRequestStream()) { stream.Write(reqArr, 0, reqArr.Length); } string response = ""; using (var clientRes = (HttpWebResponse)clientReq.GetResponse()) { using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8)) { response = readStream.ReadToEnd(); readStream.Close(); clientRes.Close(); } } var rep = JsonConvert.DeserializeObject(response); if (!rep.Status) return false; if (rep.Exist && rep.DataSize == (ulong)fileStruct.Length) return true; if (!rep.Exist) log.Info("API: Data not exist."); if (rep.Exist && rep.DataSize != (ulong)fileStruct.Length) log.Info("API: Wrong size data."); } { log.Info("Send data to remote API server."); var req = JsonConvert.SerializeObject( new DataCreateApi() { Status = true, DateAndTime = dateAndTime, Name = fileName, Struct = fileStruct }); var reqArr = Encoding.UTF8.GetBytes(req); HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/data/create"); clientReq.ContentLength = reqArr.Length; using (var stream = clientReq.GetRequestStream()) { stream.Write(reqArr, 0, reqArr.Length); } string response = ""; using (var clientRes = (HttpWebResponse)clientReq.GetResponse()) { using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8)) { response = readStream.ReadToEnd(); readStream.Close(); clientRes.Close(); } } var rep = JsonConvert.DeserializeObject(response); if (!rep.Status) return false; } return true; } catch (Exception e) { log.Warn(e.Message); return false; } } static void LogConf() { var conf = new LoggingConfiguration(); var logcon = new ConsoleTarget() { Name = "logcon", Layout = @"${time}|${level:uppercase=true}|${logger}|${message}${when:when=length('${exception}')>0:Inner=|}${exception:format=ToString,StackTrace}" }; conf.AddRule(LogLevel.Trace, LogLevel.Fatal, logcon); LogManager.Configuration = conf; } static bool SaveCurrData(string name, DateTime currDate) { try { var fileForSave = Path.Combine(Directory.GetCurrentDirectory(), "currentState"); var currStrings = Array.Empty(); if (File.Exists(fileForSave)) currStrings = File.ReadAllLines(fileForSave, Encoding.UTF8); var newString = name + ':' + currDate.Year.ToString("D4") + currDate.Month.ToString("D2") + currDate.Day.ToString("D2"); for (var i = 0; i < currStrings.Length; i++) { var splitString = currStrings[i].Split(':'); if (splitString.Length != 2) continue; if (splitString[0] != name) continue; currStrings[i] = newString; newString = null; break; } if (!string.IsNullOrEmpty(newString)) { Array.Resize(ref currStrings, currStrings.Length + 1); currStrings[currStrings.Length - 1] = newString; } File.WriteAllLines(fileForSave, currStrings, Encoding.UTF8); return true; } catch (Exception e) { log.Warn(e.Message); return false; } } static DateTime? GetCurrData(string name) { try { var file = Path.Combine(Directory.GetCurrentDirectory(), "currentState"); if (!File.Exists(file)) { log.Info("State file not exist."); return null; } var listStrings = File.ReadAllLines(file, Encoding.UTF8); for(var i = 0; i < listStrings.Length; i++) { var splitString = listStrings[i].Split(':'); if (splitString.Length != 2) continue; if (splitString[0] != name) continue; if ( !int.TryParse(splitString[1].Substring(0, 4), out int year) || !int.TryParse(splitString[1].Substring(4, 2), out int month) || !int.TryParse(splitString[1].Substring(6, 2), out int day)) { log.Warn("Wron format state string."); return null; } return new DateTime(year, month, day); } log.Info("State not exist."); return null; } catch (Exception e) { log.Warn(e.Message); return null; } } } }