노현종

분석 서버 자동화 완성

...@@ -35,7 +35,7 @@ namespace VulnCrawler ...@@ -35,7 +35,7 @@ namespace VulnCrawler
35 AWS.Account account = AWS.account; 35 AWS.Account account = AWS.account;
36 36
37 /* AWS 정보 출력 */ 37 /* AWS 정보 출력 */
38 - Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}"); 38 + Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}");
39 try 39 try
40 { 40 {
41 /* DB 접속 시도 */ 41 /* DB 접속 시도 */
......
...@@ -41,14 +41,11 @@ namespace VulnCrawler ...@@ -41,14 +41,11 @@ namespace VulnCrawler
41 if (string.IsNullOrEmpty(cve)) { 41 if (string.IsNullOrEmpty(cve)) {
42 continue; 42 continue;
43 } 43 }
44 -
45 string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}"; 44 string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}";
46 45
47 foreach (var parent in commit.Parents) { 46 foreach (var parent in commit.Parents) {
48 try 47 try
49 { 48 {
50 -
51 -
52 // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴 49 // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴
53 var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree); 50 var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree);
54 51
...@@ -132,7 +129,7 @@ namespace VulnCrawler ...@@ -132,7 +129,7 @@ namespace VulnCrawler
132 }; 129 };
133 130
134 /* VulnDB에 추가 */ 131 /* VulnDB에 추가 */
135 - VulnRDS._InsertVulnData(vuln); 132 + //VulnRDS._InsertVulnData(vuln);
136 133
137 } 134 }
138 } 135 }
......
...@@ -11,7 +11,7 @@ using System.Threading.Tasks; ...@@ -11,7 +11,7 @@ using System.Threading.Tasks;
11 using VulnCrawler; 11 using VulnCrawler;
12 using Newtonsoft.Json; 12 using Newtonsoft.Json;
13 using Newtonsoft.Json.Linq; 13 using Newtonsoft.Json.Linq;
14 - 14 +using LibGit2Sharp;
15 15
16 namespace VulnUserCodeAnalyzer 16 namespace VulnUserCodeAnalyzer
17 { 17 {
...@@ -42,7 +42,12 @@ namespace VulnUserCodeAnalyzer ...@@ -42,7 +42,12 @@ namespace VulnUserCodeAnalyzer
42 public static void AutoLoad() 42 public static void AutoLoad()
43 { 43 {
44 var dir = new DirectoryInfo(@"c:\CVE"); 44 var dir = new DirectoryInfo(@"c:\CVE");
45 - 45 + if (!dir.Exists)
46 + {
47 + Console.WriteLine("found not CVE Directory");
48 + return;
49 + }
50 + Console.WriteLine("Loading CVE List...");
46 foreach (var json in dir.EnumerateFiles("*.json")) 51 foreach (var json in dir.EnumerateFiles("*.json"))
47 { 52 {
48 var match = Regex.Match(json.Name, @"(20\d\d)"); 53 var match = Regex.Match(json.Name, @"(20\d\d)");
...@@ -59,7 +64,7 @@ namespace VulnUserCodeAnalyzer ...@@ -59,7 +64,7 @@ namespace VulnUserCodeAnalyzer
59 var dict = LoadCveJson(int.Parse(match.Value)); 64 var dict = LoadCveJson(int.Parse(match.Value));
60 CveDict.Add(year, dict); 65 CveDict.Add(year, dict);
61 66
62 - Console.WriteLine($"cve 로드 완료 {year}, 개수 : {CveDict[year].Count}"); 67 + Console.WriteLine($"Finished loading CVE List Year: {year}, Count : {CveDict[year].Count}");
63 68
64 } 69 }
65 } 70 }
...@@ -109,11 +114,50 @@ namespace VulnUserCodeAnalyzer ...@@ -109,11 +114,50 @@ namespace VulnUserCodeAnalyzer
109 } 114 }
110 class Program 115 class Program
111 { 116 {
117 + /// <summary>
118 + /// Clone 콜백 함수
119 + /// </summary>
120 + /// <param name="progress"></param>
121 + /// <returns></returns>
122 + public static bool TransferProgress(TransferProgress progress)
123 + {
124 + int totalBytes = progress.TotalObjects;
125 + int receivedBytes = progress.ReceivedObjects;
126 + long receivedTotal = progress.ReceivedBytes;
127 + double received = progress.ReceivedBytes / 1000000;
128 + double percent = ((double)receivedBytes / (double)totalBytes);
129 +
130 + Console.WriteLine($"Progress: {percent.ToString("P2")}, Remain: {receivedBytes} of {totalBytes}"); //, 받은 용량: {received.ToString()}MB");
131 + Console.ForegroundColor = ConsoleColor.DarkGreen;
132 + return true;
133 + }
134 +
135 +
136 + public static void CheckoutProcess(string path, int completedSteps, int totalSteps)
137 + {
138 + Console.WriteLine($"{completedSteps}, {totalSteps}, {path}");
139 + }
140 +
141 +
142 + public static void Clone(string path, string url)
143 + {
144 +
145 +
146 + Console.WriteLine($"Start Cloning Path : {path}");
147 +
148 +
149 + string clone = Repository.Clone(url, $@"{path}", new CloneOptions { OnTransferProgress = TransferProgress, OnCheckoutProgress = CheckoutProcess });
150 + Console.ResetColor();
151 + Console.WriteLine($"Finished Clone Repository: {clone}");
152 +
153 +
154 + }
112 static void Main(string[] args) 155 static void Main(string[] args)
113 { 156 {
114 157
158 +
115 /* 연도별 CVE JSON 파일 로드 */ 159 /* 연도별 CVE JSON 파일 로드 */
116 - //CVE_JSON.AutoLoad(); 160 + CVE_JSON.AutoLoad();
117 161
118 /* 크롤러 타입 */ 162 /* 크롤러 타입 */
119 var crawler = new VulnC(); 163 var crawler = new VulnC();
...@@ -129,7 +173,7 @@ namespace VulnUserCodeAnalyzer ...@@ -129,7 +173,7 @@ namespace VulnUserCodeAnalyzer
129 AWS.LoadAccount(xml); 173 AWS.LoadAccount(xml);
130 AWS.Account account = AWS.account; 174 AWS.Account account = AWS.account;
131 /* AWS 정보 출력 */ 175 /* AWS 정보 출력 */
132 - Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}"); 176 + Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}");
133 try 177 try
134 { 178 {
135 /* DB 접속 시도 */ 179 /* DB 접속 시도 */
...@@ -137,35 +181,71 @@ namespace VulnUserCodeAnalyzer ...@@ -137,35 +181,71 @@ namespace VulnUserCodeAnalyzer
137 } 181 }
138 catch (Exception e) 182 catch (Exception e)
139 { 183 {
140 - Console.WriteLine($"접속 에러 :: {e.ToString()}"); 184 + Console.WriteLine($"Connection Error :: {e.ToString()}");
141 return; 185 return;
142 } 186 }
143 /* AWS 연결 여부 확인 */ 187 /* AWS 연결 여부 확인 */
144 if (VulnRDS.Conn.State == System.Data.ConnectionState.Open) 188 if (VulnRDS.Conn.State == System.Data.ConnectionState.Open)
145 { 189 {
146 - Console.WriteLine("접속 성공"); 190 + Console.WriteLine("Connection Success");
147 } 191 }
148 else 192 else
149 { 193 {
150 - Console.WriteLine("연결 실패"); 194 + Console.WriteLine("Fail Connection");
151 return; 195 return;
152 } 196 }
153 197
198 + while (true)
199 + {
200 + string userId = string.Empty;
201 + string repoPath = string.Empty;
202 + Stopwatch repoWatch = new Stopwatch();
203 + repoWatch.Start();
204 + while (true)
205 + {
206 + var elapsedSeconds = repoWatch.Elapsed.TotalSeconds;
207 + if (elapsedSeconds < 10)
208 + {
209 + continue;
210 + }
211 + Console.WriteLine("Checking User DB...");
154 var reposits = VulnRDS.SelectAllReposit(); 212 var reposits = VulnRDS.SelectAllReposit();
155 -
156 foreach (var (userName, repository) in reposits) 213 foreach (var (userName, repository) in reposits)
157 { 214 {
158 - Console.WriteLine($"{userName}, {repository}"); 215 + if (string.IsNullOrWhiteSpace(repository))
216 + {
217 + continue;
159 } 218 }
219 + var repoBytes = Encoding.Unicode.GetBytes(repository);
220 + var repoBase64 = Convert.ToBase64String(repoBytes);
160 221
161 - Console.ReadLine(); 222 + var repoDir = new DirectoryInfo($@"C:\Repo\{repoBase64}");
223 + if (repoDir.Exists)
224 + {
225 + continue;
226 + }
227 + repoDir.Create();
228 + Console.WriteLine($"Clone... Path : {repoDir.FullName}, Url : {repository}");
229 + Clone(repoDir.FullName, repository);
230 +
231 + repoPath = repoDir.FullName;
232 + userId = userName;
233 + }
234 + if (!string.IsNullOrWhiteSpace(repoPath) && !string.IsNullOrWhiteSpace(userId))
235 + {
236 + break;
237 + }
238 + repoWatch.Restart();
239 + }
240 + //Console.WriteLine("엔터를 누르세요");
241 + //Console.ReadLine();
162 242
163 /* hashDict = 사용된 사용자 함수 정보 */ 243 /* hashDict = 사용된 사용자 함수 정보 */
164 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>(); 244 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
165 /* 경과 시간 체크 */ 245 /* 경과 시간 체크 */
166 Stopwatch stopwatch = new Stopwatch(); 246 Stopwatch stopwatch = new Stopwatch();
167 stopwatch.Start(); 247 stopwatch.Start();
168 - DirectoryInfo dirInfo = new DirectoryInfo(@"C:\code"); 248 + DirectoryInfo dirInfo = new DirectoryInfo(repoPath);
169 249
170 /* 모든 .c 파일 탐색 */ 250 /* 모든 .c 파일 탐색 */
171 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories); 251 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
...@@ -217,7 +297,7 @@ namespace VulnUserCodeAnalyzer ...@@ -217,7 +297,7 @@ namespace VulnUserCodeAnalyzer
217 * CVE를 가지고 있다고 인정하는 프로그램 정책 때문 297 * CVE를 가지고 있다고 인정하는 프로그램 정책 때문
218 */ 298 */
219 var searchedCveHashList = VulnRDS.SelectVulnbyCve(cve); 299 var searchedCveHashList = VulnRDS.SelectVulnbyCve(cve);
220 - Console.WriteLine($"cve:{cve}, {searchedCveHashList.Count()}개 가져옴"); 300 + Console.WriteLine($"CVE:{cve}, Received Count : {searchedCveHashList.Count()}");
221 foreach (var s in searchedCveHashList) 301 foreach (var s in searchedCveHashList)
222 { 302 {
223 vulnHashSet.Add(s); 303 vulnHashSet.Add(s);
...@@ -231,7 +311,7 @@ namespace VulnUserCodeAnalyzer ...@@ -231,7 +311,7 @@ namespace VulnUserCodeAnalyzer
231 /* 본격적인 취약점 매칭 부분 */ 311 /* 본격적인 취약점 매칭 부분 */
232 foreach (var vulnSet in vulnDict) 312 foreach (var vulnSet in vulnDict)
233 { 313 {
234 - //Console.WriteLine($"-----cve:{vulnSet.Key}"); 314 + Console.WriteLine($"-----cve:{vulnSet.Key}");
235 bool match = false; 315 bool match = false;
236 foreach (var vuln in vulnSet.Value) 316 foreach (var vuln in vulnSet.Value)
237 { 317 {
...@@ -243,6 +323,7 @@ namespace VulnUserCodeAnalyzer ...@@ -243,6 +323,7 @@ namespace VulnUserCodeAnalyzer
243 { 323 {
244 if (hashDict.ContainsKey(vuln.LenFunc)) 324 if (hashDict.ContainsKey(vuln.LenFunc))
245 { 325 {
326 + //Console.WriteLine("찾음");
246 /* Bloom Filter는 아쉽게도 포함 여부만 알 수 있기에 327 /* Bloom Filter는 아쉽게도 포함 여부만 알 수 있기에
247 * 포함되었음을 알았다면 검색해서 정보를 구한다. */ 328 * 포함되었음을 알았다면 검색해서 정보를 구한다. */
248 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash); 329 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
...@@ -271,13 +352,13 @@ namespace VulnUserCodeAnalyzer ...@@ -271,13 +352,13 @@ namespace VulnUserCodeAnalyzer
271 /* 취약점 레코드가 전부 있어야 CVE 찾음 인정 */ 352 /* 취약점 레코드가 전부 있어야 CVE 찾음 인정 */
272 if (match) 353 if (match)
273 { 354 {
274 - Console.WriteLine($"CVE 찾음 {vulnSet.Key}"); 355 + Console.WriteLine($"Matched CVE : {vulnSet.Key}");
275 /* 찾았으면 cve값을 기록함 밑에서 찾은 cve 정보 전송하기 위해 */ 356 /* 찾았으면 cve값을 기록함 밑에서 찾은 cve 정보 전송하기 위해 */
276 findCveList.Add(vulnSet.Key); 357 findCveList.Add(vulnSet.Key);
277 } 358 }
278 else 359 else
279 { 360 {
280 - Console.WriteLine("없음"); 361 + Console.WriteLine("Not");
281 } 362 }
282 } 363 }
283 stopwatch.Stop(); 364 stopwatch.Stop();
...@@ -285,8 +366,10 @@ namespace VulnUserCodeAnalyzer ...@@ -285,8 +366,10 @@ namespace VulnUserCodeAnalyzer
285 var hours = stopwatch.Elapsed.Hours; 366 var hours = stopwatch.Elapsed.Hours;
286 var minutes = stopwatch.Elapsed.Minutes; 367 var minutes = stopwatch.Elapsed.Minutes;
287 var seconds = stopwatch.Elapsed.Seconds; 368 var seconds = stopwatch.Elapsed.Seconds;
288 - Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}"); 369 + Console.WriteLine($"Elapsed Time : {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
289 - Console.WriteLine($"찾은 CVE 개수 : {findCveList.Count}"); 370 + Console.WriteLine($"Matched CVE Count : {findCveList.Count}");
371 + //Console.ReadLine();
372 +
290 var yearMatch = new Regex(@"CVE-(\d{4})-(\d+)"); 373 var yearMatch = new Regex(@"CVE-(\d{4})-(\d+)");
291 foreach (var cve in findCveList) 374 foreach (var cve in findCveList)
292 { 375 {
...@@ -340,9 +423,9 @@ namespace VulnUserCodeAnalyzer ...@@ -340,9 +423,9 @@ namespace VulnUserCodeAnalyzer
340 423
341 var urlBytes = Convert.FromBase64String(findCveDict[cve].FirstOrDefault().Url); 424 var urlBytes = Convert.FromBase64String(findCveDict[cve].FirstOrDefault().Url);
342 string url = Encoding.Unicode.GetString(urlBytes); 425 string url = Encoding.Unicode.GetString(urlBytes);
426 + //Console.WriteLine(findCveDict[cve].FirstOrDefault().Path.Replace(repoPath, ""));
343 427
344 - /* DB 전송 */ 428 + var vulnDetail = new VulnRDS.Vuln_detail
345 - VulnRDS.InsertVulnDetail(new VulnRDS.Vuln_detail
346 { 429 {
347 CveName = data.Code, 430 CveName = data.Code,
348 Type = type, 431 Type = type,
...@@ -351,13 +434,18 @@ namespace VulnUserCodeAnalyzer ...@@ -351,13 +434,18 @@ namespace VulnUserCodeAnalyzer
351 CveDetail = data.Detail, 434 CveDetail = data.Detail,
352 Publish_date = data.Publish_Date.ToString("yyyy-MM-dd"), 435 Publish_date = data.Publish_Date.ToString("yyyy-MM-dd"),
353 Update_date = data.Update_Date.ToString("yyyy-MM-dd"), 436 Update_date = data.Update_Date.ToString("yyyy-MM-dd"),
354 - UserName = "samsung", 437 + UserName = userId,
355 Url = url, 438 Url = url,
356 - FileName = findCveDict[cve].FirstOrDefault().Path.Replace(@"C:\code", ""), 439 + FileName = findCveDict[cve].FirstOrDefault().Path.Replace(repoPath, ""),
357 FuncName = findCveDict[cve].FirstOrDefault().FuncName, 440 FuncName = findCveDict[cve].FirstOrDefault().FuncName,
358 Product = data.Type, 441 Product = data.Type,
359 - }); 442 + };
360 - Console.WriteLine("추가 완료"); 443 +
444 + /* DB 전송 */
445 + VulnRDS.InsertVulnDetail(vulnDetail);
446 +
447 + Console.WriteLine($"Added CVE: {vulnDetail.CveName}, Type: {vulnDetail.Type}, CVSS: {vulnDetail.Level}");
448 + }
361 } 449 }
362 } 450 }
363 } 451 }
......
1 <?xml version="1.0" encoding="utf-8"?> 1 <?xml version="1.0" encoding="utf-8"?>
2 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> 2 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3 + <Import Project="..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props" Condition="Exists('..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props')" />
3 <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> 4 <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
4 <PropertyGroup> 5 <PropertyGroup>
5 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> 6 <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
...@@ -11,6 +12,8 @@ ...@@ -11,6 +12,8 @@
11 <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion> 12 <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
12 <FileAlignment>512</FileAlignment> 13 <FileAlignment>512</FileAlignment>
13 <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects> 14 <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
15 + <NuGetPackageImportStamp>
16 + </NuGetPackageImportStamp>
14 </PropertyGroup> 17 </PropertyGroup>
15 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> 18 <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
16 <PlatformTarget>AnyCPU</PlatformTarget> 19 <PlatformTarget>AnyCPU</PlatformTarget>
...@@ -32,6 +35,9 @@ ...@@ -32,6 +35,9 @@
32 <WarningLevel>4</WarningLevel> 35 <WarningLevel>4</WarningLevel>
33 </PropertyGroup> 36 </PropertyGroup>
34 <ItemGroup> 37 <ItemGroup>
38 + <Reference Include="LibGit2Sharp, Version=0.25.0.0, Culture=neutral, PublicKeyToken=7cbde695407f0333, processorArchitecture=MSIL">
39 + <HintPath>..\packages\LibGit2Sharp.0.25.2\lib\netstandard2.0\LibGit2Sharp.dll</HintPath>
40 + </Reference>
35 <Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" /> 41 <Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" />
36 <Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL"> 42 <Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
37 <HintPath>..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll</HintPath> 43 <HintPath>..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
...@@ -64,4 +70,10 @@ ...@@ -64,4 +70,10 @@
64 </ProjectReference> 70 </ProjectReference>
65 </ItemGroup> 71 </ItemGroup>
66 <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> 72 <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
73 + <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
74 + <PropertyGroup>
75 + <ErrorText>이 프로젝트는 이 컴퓨터에 없는 NuGet 패키지를 참조합니다. 해당 패키지를 다운로드하려면 NuGet 패키지 복원을 사용하십시오. 자세한 내용은 http://go.microsoft.com/fwlink/?LinkID=322105를 참조하십시오. 누락된 파일은 {0}입니다.</ErrorText>
76 + </PropertyGroup>
77 + <Error Condition="!Exists('..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props'))" />
78 + </Target>
67 </Project> 79 </Project>
...\ No newline at end of file ...\ No newline at end of file
......
1 <?xml version="1.0" encoding="utf-8"?> 1 <?xml version="1.0" encoding="utf-8"?>
2 <packages> 2 <packages>
3 + <package id="LibGit2Sharp" version="0.25.2" targetFramework="net461" />
4 + <package id="LibGit2Sharp.NativeBinaries" version="1.0.217" targetFramework="net461" />
3 <package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" /> 5 <package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" />
4 <package id="System.ValueTuple" version="4.5.0" targetFramework="net461" /> 6 <package id="System.ValueTuple" version="4.5.0" targetFramework="net461" />
5 </packages> 7 </packages>
...\ No newline at end of file ...\ No newline at end of file
......