ASCU_ALL/ClientCollector/Program.cs

using NLog;
using NLog.Config;
using NLog.Targets;
using System;
using System.Net;
using System.Net.Http;
using DataClient;
using DataClient.Struct;
using System.Linq;
using System.Threading.Tasks;
using System.Net.Http.Headers;
using ApiServer.ApiStruct;
using Newtonsoft.Json;
using System.Text;
using System.IO;
using System.Collections.Generic;
using System.Diagnostics;
using ICSharpCode.SharpZipLib.GZip;
using ICSharpCode.SharpZipLib.Tar;
using System.Net.Security;

namespace ClientCollector
{
	class Program
	{
		static Logger log = LogManager.GetCurrentClassLogger();

		static void Main(string[] args)
		{
			LogConf();
			log.Info("Start Client Collector.");
			Task taskPasp = null;
			var workPasp = false;
			Task taskData = null;
			var workData = false;

			foreach(var arg in args)
			{
				if(int.TryParse(arg, out int flag))
					switch (flag)
					{
						case 1:
							workPasp = true;
							break;
						case 2:
							workData = true;
							break;
					}
			}

			while (true)
			{
				if (taskPasp == null || taskPasp.IsCompleted)
				{
					log.Info(workPasp ? "Start Pasport Task." : "Skip Pasport Task.");
					taskPasp = workPasp ? WorkPasport() : Task.Delay(int.MaxValue);
				}
				if (taskData == null || taskData.IsCompleted)
				{
					log.Info(workData ? "Start Data Task." : "Skip Data Task.");
					taskData = workData ? WorkData() : Task.Delay(int.MaxValue);
				}
				log.Info("Wait tasks.");
				Task.WaitAny(new Task[] { taskPasp, taskData });
			}
		}

		static async Task WorkPasport()
		{
			await Task.Delay(1000);
			var IpSTP = "10.10.45.152";
			var PortSTP = 1070;
			var nameCurrDate = "pasport";
			var currDate = GetCurrData(nameCurrDate);
			if (!currDate.HasValue)
				currDate = new DateTime(2001, 02, 01);
			NETClient netClient = new NETClient(IpSTP, PortSTP);
			var flagcycle = false;
			while (currDate.Value < DateTime.Now.AddDays(-1))
			{
				try
				{
					var currDir =
						currDate.Value.Year.ToString("D4") + '/' +
						currDate.Value.Month.ToString("D2") + '/' +
						currDate.Value.Day.ToString("D2");
					log.Info("Get pasports from: " + currDir);
					while (!netClient.Connected() || flagcycle)
					{
						flagcycle = false;
						if (!netClient.ReConnect())
						{
							netClient.Close();
							netClient = new NETClient(IpSTP, PortSTP);
							log.Warn("Can't connect to STP.");
							await Task.Delay(10000);
						}
					}
					var currPasports = netClient.Full_Dir_Browse(currDir);
					netClient.Close();
					if (currPasports == null)
						currPasports = Array.Empty<string>();
					foreach (var paspDir in currPasports)
					{
						do
						{
							try
							{
								log.Info("Get pasport: " + paspDir);
								while (!netClient.Connected() || flagcycle)
								{
									flagcycle = false;
									if (!netClient.ReConnect())
									{
										log.Warn("Can't connect to STP.");
										netClient.Close();
										netClient = new NETClient(IpSTP, PortSTP);
										await Task.Delay(10000);
									}
								}
								var pasp = netClient.Full_Pasp_Download(paspDir);
								netClient.Close();
								if (pasp == null || !pasp.HasData)
								{
									log.Warn("Can't get pasport.");
									netClient.Close();
									netClient = new NETClient(IpSTP, PortSTP);
									flagcycle = true;
									await Task.Delay(10000);
									continue;
								}
								while (!SendPasport(pasp))
								{
									log.Warn("Can't send pasp to API.");
									await Task.Delay(10000);
								}
								break;
							}
							catch (Exception e)
							{
								log.Warn(e.Message);
								netClient.Close();
								flagcycle = true;
								await Task.Delay(5000);
							}
						}
						while (true);
					}
					log.Info("End day: " + currDir);
					SaveCurrData(nameCurrDate, currDate.Value);
					currDate = currDate.Value.AddDays(1);
					await Task.Delay(2000);
				}
				catch (Exception e)
				{
					log.Warn(e.Message);
					flagcycle = true;
					netClient.Close();
					netClient = new NETClient(IpSTP, PortSTP);
					await Task.Delay(1000 * 60 * 5);
				}
			}
			do
			{
				log.Info("Wait next day.");
				await Task.Delay(1000 * 60 * 60);
			} while (!(currDate.Value < DateTime.Now.AddDays(-1)));
			currDate.Value.AddDays(-15);
			SaveCurrData(nameCurrDate, currDate.Value);
		}

		static async Task WorkData()
		{
			await Task.Delay(1000);
			var IpSTP = "10.10.45.152";
			var PortSTP = 1070;
			var mainDir = Path.Combine(Directory.GetCurrentDirectory(), "archive");
			var nameCurrDate = "data";
			var currDate = GetCurrData(nameCurrDate);
			if (!currDate.HasValue)
				currDate = new DateTime(2001, 02, 01);
			while (currDate.Value < DateTime.Now.AddDays(-1))
			{
				try
				{
					while(DateTime.Now.Hour < 8)
          {
						log.Info("Await 8:00.");
						await Task.Delay(15 * 1000 * 60);
          }
					log.Info("Disconnect archive.");
					while (DisconnectNFS(mainDir))
						await Task.Delay(1000);
					log.Info("Connect to archive.");
					while (!ConnectNFS(mainDir))
						await Task.Delay(1000);

					var subDir = Path.Combine(mainDir, "data",
						currDate.Value.Year.ToString("D4"),
						currDate.Value.Month.ToString("D2"),
						currDate.Value.Day.ToString("D2"));
					log.Info("Check dir: " + subDir);
					if (Directory.Exists(Path.Combine(mainDir, "data"))) {
						if (Directory.Exists(subDir))
						{
							var listFiles = Directory.GetFiles(subDir);
							foreach (var fileDir in listFiles)
							{
								log.Info("Get file: " + fileDir);
								var fileStruct = File.ReadAllBytes(fileDir);
								log.Info("Send file to API: " + fileDir);
								while (!SendData(currDate.Value, Path.GetFileName(fileDir), fileStruct))
								{
									log.Warn("Can't send data to API.");
									await Task.Delay(10000);
								}
							}
						}
						else
						{
							var listVDP = new List<int>();
							for (var i = 0; i < 50; i++)  listVDP.Add(i);
							for (var i = 90; i < 95; i++) listVDP.Add(i);
							foreach (var vdp in listVDP)
							{
								var pathTmp = Path.Combine(Directory.GetCurrentDirectory(), "tmpData");
								if (Directory.Exists(pathTmp)) Directory.Delete(pathTmp, true);
								Directory.CreateDirectory(pathTmp);
								for (var numFile = 0; numFile < 16; numFile++)
								{
									await Task.Delay(2000);
									var name = currDate.Value.ToString("yyyyMMdd") + "." + vdp.ToString("D2") + numFile.ToString("X1");
									try
									{
										var nc = new NETClient(IpSTP, PortSTP);
										while (!nc.Connected())
											if (!nc.ReConnect())
											{
												log.Warn("Can't connect to STP.");
												await Task.Delay(5000);
											}
										log.Info("Try download " + name);
										var array = nc.Full_Download_NH(currDate.Value, vdp, numFile);
										nc.Close();
										if (array == null || array.Length == 0)
										{
											log.Info("File " + name + " not exist.");
											continue;
										}
										File.WriteAllBytes(Path.Combine(pathTmp, name), array);
									}
									catch (Exception e)
									{
										log.Warn(e.Message);
										log.Warn("Can't download " + name);
									}
								}
								{
									var files = Directory.GetFiles(pathTmp);
									if (files.Length > 0)
									{
										var targzFile = vdp.ToString("D2") + ".tar.gz";
										using (var outStream = File.Create(Path.Combine(pathTmp, targzFile)))
										using (var gzoStream = new GZipOutputStream(outStream))
										using (var tarArchive = TarArchive.CreateOutputTarArchive(gzoStream))
											foreach (var file in files)
											{
												tarArchive.RootPath = Path.GetDirectoryName(file);
												var tarEntry = TarEntry.CreateEntryFromFile(file);
												tarEntry.Name = Path.GetFileName(file);
												tarArchive.WriteEntry(tarEntry, true);
											}
										var fileStruct = File.ReadAllBytes(Path.Combine(pathTmp, targzFile));
										while (!SendData(currDate.Value, Path.GetFileName(targzFile), fileStruct))
										{
											log.Warn("Can't send data to API.");
											await Task.Delay(10000);
										}
									}
									Directory.Delete(pathTmp, true);
								}
							}
						}
						log.Info("End day: " + subDir);
						SaveCurrData(nameCurrDate, currDate.Value);
						currDate = currDate.Value.AddDays(1);
					}
					else
					{
						log.Warn("Archive not exist.");

					}
				}
				catch (Exception e)
				{
					log.Warn(e.Message);
				}
			}
			do
			{
				log.Info("Wait next day.");
				await Task.Delay(1000 * 60 * 60);
			} while (!(currDate.Value < DateTime.Now.AddDays(-1)));
			currDate.Value.AddDays(-15);
			SaveCurrData(nameCurrDate, currDate.Value);
		}

		static bool ConnectNFS(string dir)
		{
			var dirCreated = false;
			while (!dirCreated)
			{
				try
				{
					dirCreated = Directory.Exists(Path.Combine(dir));
					if (!dirCreated) Directory.CreateDirectory(dir);
				}
				catch (Exception e)
				{
					log.Warn(e.Message);
					log.Warn("Can't create directory");
					Task.Delay(60000).Wait();
				}
			}
			try
			{
				var procC = new Process
				{
					StartInfo = new ProcessStartInfo
					{
						FileName = "/usr/bin/mount",
						Arguments = "-t nfs 10.10.45.236:/archiv " + dir,
						UseShellExecute = false,
						RedirectStandardError = true,
						RedirectStandardOutput = true,
					},
				};
				if (!procC.Start())
				{
					log.Warn("Can't start mount command.");
					Task.Delay(60000).Wait();
				}
				var answer = procC.StandardError.ReadToEnd() + procC.StandardOutput.ReadToEnd();
				var connected = string.IsNullOrEmpty(answer);
				if (connected)
				{
					log.Info("Connect Success");
					return true;
				}
				log.Info(answer);
				return false;
			}
			catch (Exception e)
			{
				log.Warn(e.Message);
				log.Warn("Can't connect to NFS.");
				Task.Delay(60000).Wait();
				return false;
			}

		}
		static bool DisconnectNFS(string dir)
		{
			var dirCreated = false;
			while (!dirCreated)
			{
				try
				{
					dirCreated = Directory.Exists(Path.Combine(dir));
					if (!dirCreated) Directory.CreateDirectory(dir);
				}
				catch (Exception e)
				{
					log.Warn(e.Message);
					log.Warn("Can't create directory");
					Task.Delay(60000).Wait();
				}
			}
			try
			{
				var procD = new Process
				{
					StartInfo = new ProcessStartInfo
					{
						FileName = "/usr/bin/umount",
						Arguments = dir,
						UseShellExecute = false,
						RedirectStandardError = true,
						RedirectStandardOutput = true,
					}
				};
				if (!procD.Start())
				{
					log.Warn("Can't start umount command.");
					Task.Delay(60000).Wait();
				}
				var answer = procD.StandardError.ReadToEnd() + procD.StandardOutput.ReadToEnd();
				log.Info(string.IsNullOrEmpty(answer) ? "NFS disconnected." : answer);
				return string.IsNullOrEmpty(answer);
			}
			catch (Exception e)
			{
				log.Warn(e.Message);
				return false;
			}
		}


		static HttpWebRequest GetRequest(string path, bool useProxy = true)
		{
			HttpWebRequest clientReq = (HttpWebRequest)WebRequest.Create(path);
			clientReq.ServerCertificateValidationCallback = (sender, cert, chain, error) => { return true; };
			clientReq.Method = "POST";
			clientReq.ContentType = "application/json; charset=utf-8";
			if (useProxy)
			{
				clientReq.Proxy = new WebProxy("194.226.128.245", 3128)
				{
					BypassProxyOnLocal = false,
					Credentials = new NetworkCredential("user4", "user4")
				};
			}
			return clientReq;
		}
		static bool SendPasport(Pasport pasp)
		{
			try
			{
				{
					log.Info("Check pasport on remote API server.");
					var req = JsonConvert.SerializeObject(
						new PasportCheckApi()
						{
							Status = true,
							DateAndTime = pasp.dEnd.Value,
							Name = string.IsNullOrEmpty(pasp.nplav) ? pasp.numVDP.Value.ToString("D2") : pasp.nplav
						});
					var reqArr = Encoding.UTF8.GetBytes(req);
					HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/pasport/check");
					clientReq.ContentLength = reqArr.Length;
					using (var stream = clientReq.GetRequestStream())
					{
						stream.Write(reqArr, 0, reqArr.Length);
					}
					string response = "";
					using (var clientRes = (HttpWebResponse)clientReq.GetResponse())
					{
						using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8))
						{
							response = readStream.ReadToEnd();
							readStream.Close();
							clientRes.Close();
						}
					}
					var rep = JsonConvert.DeserializeObject<PasportCheckClient>(response);
					if (!rep.Status) return false;
					ulong paspSum = 0;
					var paspArr = pasp.PaspByte;
					foreach (var b in paspArr)
						paspSum += b;
					if (rep.Exist && rep.PaspSum == paspSum) return true;
					if (!rep.Exist)
						log.Info("API: Pasport not exist.");
					if (rep.Exist && rep.PaspSum != paspSum)
						log.Info("API: Wrong size pasport.");
				}
				{
					log.Info("Send pasport to remote API server.");
					var req = JsonConvert.SerializeObject(
						new PasportCreateApi()
						{
							Status = true,
							Pasp = pasp
						});
					var reqArr = Encoding.UTF8.GetBytes(req);

					HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/pasport/create");
					clientReq.ContentLength = reqArr.Length;
					using (var stream = clientReq.GetRequestStream())
					{
						stream.Write(reqArr, 0, reqArr.Length);
					}
					string response = "";
					using (var clientRes = (HttpWebResponse)clientReq.GetResponse())
					{
						using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8))
						{
							response = readStream.ReadToEnd();
							readStream.Close();
							clientRes.Close();
						}
					}
					var rep = JsonConvert.DeserializeObject<PasportCreateClient>(response);
					if (!rep.Status) return false;
				}
				return true;
			}
			catch (Exception e)
			{
				log.Warn(e.Message);
				return false;
			}
		}
		static bool SendData(DateTime dateAndTime, string fileName, byte[] fileStruct)
    {
			try
			{
				{
					log.Info("Check data on remote API server.");
					var req = JsonConvert.SerializeObject(
						new DataCheckApi()
						{
							Status = true,
							DateAndTime = dateAndTime,
							Name = fileName
						});
					var reqArr = Encoding.UTF8.GetBytes(req);
					HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/data/check");
					clientReq.ContentLength = reqArr.Length;
					using (var stream = clientReq.GetRequestStream())
					{
						stream.Write(reqArr, 0, reqArr.Length);
					}
					string response = "";
					using (var clientRes = (HttpWebResponse)clientReq.GetResponse())
					{
						using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8))
						{
							response = readStream.ReadToEnd();
							readStream.Close();
							clientRes.Close();
						}
					}
					var rep = JsonConvert.DeserializeObject<DataCheckClient>(response);
					if (!rep.Status) return false;
					if (rep.Exist && rep.DataSize == (ulong)fileStruct.Length) return true;
					if (!rep.Exist)
						log.Info("API: Data not exist.");
					if (rep.Exist && rep.DataSize != (ulong)fileStruct.Length)
						log.Info("API: Wrong size data.");
				}
				{
					log.Info("Send data to remote API server.");
					var req = JsonConvert.SerializeObject(
						new DataCreateApi()
						{
							Status = true,
							DateAndTime = dateAndTime,
							Name = fileName,
							Struct = fileStruct
						});
					var reqArr = Encoding.UTF8.GetBytes(req);

					HttpWebRequest clientReq = GetRequest("https://vsmpo.mbucb.ru/api/data/create");
					clientReq.ContentLength = reqArr.Length;
					using (var stream = clientReq.GetRequestStream())
					{
						stream.Write(reqArr, 0, reqArr.Length);
					}
					string response = "";
					using (var clientRes = (HttpWebResponse)clientReq.GetResponse())
					{
						using (var readStream = new StreamReader(clientRes.GetResponseStream(), Encoding.UTF8))
						{
							response = readStream.ReadToEnd();
							readStream.Close();
							clientRes.Close();
						}
					}
					var rep = JsonConvert.DeserializeObject<DataCreateClient>(response);
					if (!rep.Status) return false;
				}
				return true;
			}
			catch (Exception e)
			{
				log.Warn(e.Message);
				return false;
			}
		}
		static void LogConf()
		{
			var conf = new LoggingConfiguration();
			var logcon = new ConsoleTarget()
			{
				Name = "logcon",
				Layout = @"${time}|${level:uppercase=true}|${logger}|${message}${when:when=length('${exception}')>0:Inner=|}${exception:format=ToString,StackTrace}"
			};

			conf.AddRule(LogLevel.Trace, LogLevel.Fatal, logcon);
			LogManager.Configuration = conf;
		}
		static bool SaveCurrData(string name, DateTime currDate)
		{
			try
			{
				var fileForSave = Path.Combine(Directory.GetCurrentDirectory(), "currentState");
				var currStrings = Array.Empty<string>();
				if (File.Exists(fileForSave))
					currStrings = File.ReadAllLines(fileForSave, Encoding.UTF8);
				var newString =
					name + ':' +
					currDate.Year.ToString("D4") +
					currDate.Month.ToString("D2") +
					currDate.Day.ToString("D2");
				for (var i = 0; i < currStrings.Length; i++)
        {
					var splitString = currStrings[i].Split(':');
					if (splitString.Length != 2) continue;
					if (splitString[0] != name) continue;
					currStrings[i] = newString;
					newString = null;
					break;
				}
				if (!string.IsNullOrEmpty(newString))
        {
					Array.Resize(ref currStrings, currStrings.Length + 1);
					currStrings[currStrings.Length - 1] = newString;
        }
				File.WriteAllLines(fileForSave, currStrings, Encoding.UTF8);
				return true;
			}
			catch (Exception e)
			{
				log.Warn(e.Message);
				return false;
			}
		}
		static DateTime? GetCurrData(string name)
		{
			try
			{
				var file = Path.Combine(Directory.GetCurrentDirectory(), "currentState");
				if (!File.Exists(file))
				{
					log.Info("State file not exist.");
					return null;
				}
				var listStrings = File.ReadAllLines(file, Encoding.UTF8);
				for(var i = 0; i < listStrings.Length; i++)
        {
					var splitString = listStrings[i].Split(':');
					if (splitString.Length != 2) continue;
					if (splitString[0] != name) continue;
					if (
					!int.TryParse(splitString[1].Substring(0, 4), out int year) ||
					!int.TryParse(splitString[1].Substring(4, 2), out int month) ||
					!int.TryParse(splitString[1].Substring(6, 2), out int day))
					{
						log.Warn("Wron format state string.");
						return null;
					}
					return new DateTime(year, month, day);
				}
				log.Info("State not exist.");
				return null;
			}
			catch (Exception e)
			{
				log.Warn(e.Message);
				return null;
			}
		}

	}
}