분석 서버 자동화 완성

노현종
Commit e574b16caf67bac914bbe63c24f13c8474ff16c9 e574b16c 1 parent ddbc5576
Showing 5 changed files with 110 additions and 15 deletions
Vulnerablity_DB/VulnCrawler/Program.cs
Vulnerablity_DB/VulnCrawler/VulnWorker.cs
Vulnerablity_DB/VulnUserCodeAnalyzer/Program.cs
Vulnerablity_DB/VulnUserCodeAnalyzer/VulnUserCodeAnalyzer.csproj
Vulnerablity_DB/VulnUserCodeAnalyzer/packages.config
--- a/Vulnerablity_DB/VulnCrawler/Program.cs
View file @e574b16
+++ b/Vulnerablity_DB/VulnCrawler/Program.cs
View file @e574b16
@@ -35,7 +35,7 @@ namespace VulnCrawler
             AWS.Account account = AWS.account;
             /* AWS 정보 출력 */
-            Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
+            Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}");
             try
             {
                 /* DB 접속 시도 */
--- a/Vulnerablity_DB/VulnCrawler/VulnWorker.cs
View file @e574b16
+++ b/Vulnerablity_DB/VulnCrawler/VulnWorker.cs
View file @e574b16
@@ -41,14 +41,11 @@ namespace VulnCrawler
                 if (string.IsNullOrEmpty(cve)) {
                     continue;
                 }
-
                 string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}";
                 foreach (var parent in commit.Parents) {
                     try
                     {
-             
-
                         // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴
                         var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree);
@@ -132,7 +129,7 @@ namespace VulnCrawler
                             };
                             /* VulnDB에 추가 */
-                            VulnRDS._InsertVulnData(vuln);
+                            //VulnRDS._InsertVulnData(vuln);
                         }
                     }
--- a/Vulnerablity_DB/VulnUserCodeAnalyzer/Program.cs
View file @e574b16
+++ b/Vulnerablity_DB/VulnUserCodeAnalyzer/Program.cs
View file @e574b16
@@ -11,7 +11,7 @@ using System.Threading.Tasks;
 using VulnCrawler;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
-
+using LibGit2Sharp;
 namespace VulnUserCodeAnalyzer
 {
@@ -109,11 +109,50 @@ namespace VulnUserCodeAnalyzer
     }
     class Program
     {
+        /// <summary>
+        /// Clone 콜백 함수
+        /// </summary>
+        /// <param name="progress"></param>
+        /// <returns></returns>
+        public static bool TransferProgress(TransferProgress progress)
+        {
+            int totalBytes = progress.TotalObjects;
+            int receivedBytes = progress.ReceivedObjects;
+            long receivedTotal = progress.ReceivedBytes;
+            double received = progress.ReceivedBytes / 1000000;
+            double percent = ((double)receivedBytes / (double)totalBytes);
+
+            Console.WriteLine($"진행률: {percent.ToString("P2")}, 남은 파일: {receivedBytes} of {totalBytes}"); //, 받은 용량: {received.ToString()}MB");
+            Console.ForegroundColor = ConsoleColor.DarkGreen;
+            return true;
+        }
+
+
+        public static void CheckoutProcess(string path, int completedSteps, int totalSteps)
+        {
+            Console.WriteLine($"{completedSteps}, {totalSteps}, {path}");
+        }
+
+
+        public static void Clone(string path, string url)
+        {
+ 
+
+            Console.WriteLine($"다운로드를 진행합니다. 경로 : {path}");
+
+
+            string clone = Repository.Clone(url, $@"{path}", new CloneOptions { OnTransferProgress = TransferProgress, OnCheckoutProgress = CheckoutProcess });
+            Console.ResetColor();
+            Console.WriteLine($"Clone: {clone}");
+
+
+        }
         static void Main(string[] args)
         {
+            
             /* 연도별 CVE JSON 파일 로드 */
-            //CVE_JSON.AutoLoad();
+            CVE_JSON.AutoLoad();
             /* 크롤러 타입 */
             var crawler = new VulnC();
@@ -129,7 +168,7 @@ namespace VulnUserCodeAnalyzer
             AWS.LoadAccount(xml);
             AWS.Account account = AWS.account;
             /* AWS 정보 출력 */
-            Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
+            Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}");
             try
             {
                 /* DB 접속 시도 */
@@ -151,21 +190,62 @@ namespace VulnUserCodeAnalyzer
                 return;
             }
+            while (true)
+            {
+
+                string userId = string.Empty;
+                string repoPath = string.Empty;
+
+
+                Stopwatch repoWatch = new Stopwatch();
+                repoWatch.Start();
+                while (true)
+                {
+                    var elapsedSeconds = repoWatch.Elapsed.TotalSeconds;
+                    if (elapsedSeconds < 10)
+                    {
+                        
+                        continue;
+                    }
+                    Console.WriteLine("SelectAllReposit() 실행");
                     var reposits = VulnRDS.SelectAllReposit();
                     foreach (var (userName, repository) in reposits)
                     {
-                Console.WriteLine($"{userName}, {repository}");
+                        if (string.IsNullOrWhiteSpace(repository))
+                        {
+                            continue;
+                        }
+                        var repoBytes = Encoding.Unicode.GetBytes(repository);
+                        var repoBase64 = Convert.ToBase64String(repoBytes);
+
+                        var repoDir = new DirectoryInfo($@"C:\Repo\{repoBase64}");
+                        if (repoDir.Exists)
+                        {
+                            continue;
                         }
+                        repoDir.Create();
+                        Console.WriteLine($"Clone... Path : {repoDir.FullName}, Url : {repository}");
+                        Clone(repoDir.FullName, repository);
-            Console.ReadLine();
+                        repoPath = repoDir.FullName;
+                        userId = userName;
+                    }
+                    if (!string.IsNullOrWhiteSpace(repoPath) && !string.IsNullOrWhiteSpace(userId))
+                    {
+                        break;
+                    }
+                    repoWatch.Restart();
+                }
+                //Console.WriteLine("엔터를 누르세요");
+                //Console.ReadLine();
                 /* hashDict = 사용된 사용자 함수 정보 */
                 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
                 /* 경과 시간 체크 */
                 Stopwatch stopwatch = new Stopwatch();
                 stopwatch.Start();
-            DirectoryInfo dirInfo = new DirectoryInfo(@"C:\code");
+                DirectoryInfo dirInfo = new DirectoryInfo(repoPath);
                 /* 모든 .c 파일 탐색 */
                 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
@@ -231,7 +311,7 @@ namespace VulnUserCodeAnalyzer
                 /* 본격적인 취약점 매칭 부분 */
                 foreach (var vulnSet in vulnDict)
                 {
-                //Console.WriteLine($"-----cve:{vulnSet.Key}");
+                    Console.WriteLine($"-----cve:{vulnSet.Key}");
                     bool match = false;
                     foreach (var vuln in vulnSet.Value)
                     {
@@ -243,6 +323,7 @@ namespace VulnUserCodeAnalyzer
                         {
                             if (hashDict.ContainsKey(vuln.LenFunc))
                             {
+                                Console.WriteLine("찾음");
                                 /* Bloom Filter는 아쉽게도 포함 여부만 알 수 있기에 
                                  * 포함되었음을 알았다면 검색해서 정보를 구한다. */
                                 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
@@ -287,6 +368,8 @@ namespace VulnUserCodeAnalyzer
                 var seconds = stopwatch.Elapsed.Seconds;
                 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
                 Console.WriteLine($"찾은 CVE 개수 : {findCveList.Count}");
+                //Console.ReadLine();
+
                 var yearMatch = new Regex(@"CVE-(\d{4})-(\d+)");
                 foreach (var cve in findCveList)
                 {
@@ -340,7 +423,7 @@ namespace VulnUserCodeAnalyzer
                     var urlBytes = Convert.FromBase64String(findCveDict[cve].FirstOrDefault().Url);
                     string url = Encoding.Unicode.GetString(urlBytes);
-
+                    Console.WriteLine(findCveDict[cve].FirstOrDefault().Path.Replace(repoPath, ""));
                     /* DB 전송 */
                     VulnRDS.InsertVulnDetail(new VulnRDS.Vuln_detail
                     {
@@ -351,9 +434,9 @@ namespace VulnUserCodeAnalyzer
                         CveDetail = data.Detail,
                         Publish_date = data.Publish_Date.ToString("yyyy-MM-dd"),
                         Update_date = data.Update_Date.ToString("yyyy-MM-dd"),
-                    UserName = "samsung",
+                        UserName = userId,
                         Url = url,
-                    FileName = findCveDict[cve].FirstOrDefault().Path.Replace(@"C:\code", ""),
+                        FileName = findCveDict[cve].FirstOrDefault().Path.Replace(repoPath, ""),
                         FuncName = findCveDict[cve].FirstOrDefault().FuncName,
                         Product = data.Type,
                     });
@@ -361,5 +444,6 @@ namespace VulnUserCodeAnalyzer
                 }
             }
         }
+    }
 }
--- a/Vulnerablity_DB/VulnUserCodeAnalyzer/VulnUserCodeAnalyzer.csproj
View file @e574b16
+++ b/Vulnerablity_DB/VulnUserCodeAnalyzer/VulnUserCodeAnalyzer.csproj
View file @e574b16
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props" Condition="Exists('..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props')" />
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -11,6 +12,8 @@
     <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>AnyCPU</PlatformTarget>
@@ -32,6 +35,9 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="LibGit2Sharp, Version=0.25.0.0, Culture=neutral, PublicKeyToken=7cbde695407f0333, processorArchitecture=MSIL">
+      <HintPath>..\packages\LibGit2Sharp.0.25.2\lib\netstandard2.0\LibGit2Sharp.dll</HintPath>
+    </Reference>
     <Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" />
     <Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
@@ -64,4 +70,10 @@
     </ProjectReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>이 프로젝트는 이 컴퓨터에 없는 NuGet 패키지를 참조합니다. 해당 패키지를 다운로드하려면 NuGet 패키지 복원을 사용하십시오. 자세한 내용은 http://go.microsoft.com/fwlink/?LinkID=322105를 참조하십시오. 누락된 파일은 {0}입니다.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props'))" />
+  </Target>
 </Project>
\ No newline at end of file
--- a/Vulnerablity_DB/VulnUserCodeAnalyzer/packages.config
View file @e574b16
+++ b/Vulnerablity_DB/VulnUserCodeAnalyzer/packages.config
View file @e574b16
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="LibGit2Sharp" version="0.25.2" targetFramework="net461" />
+  <package id="LibGit2Sharp.NativeBinaries" version="1.0.217" targetFramework="net461" />
   <package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" />
   <package id="System.ValueTuple" version="4.5.0" targetFramework="net461" />
 </packages>
\ No newline at end of file