노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
노현종
ba5caddb 1 parent ddbc5576

분석 서버 자동화 완성

Inline Side-by-side
Showing 5 changed files with 128 additions and 29 deletions
......@@ -35,7 +35,7 @@ namespace VulnCrawler
AWS.Account account = AWS.account;
/* AWS 정보 출력 */
Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}");
try
{
/* DB 접속 시도 */
......
......@@ -41,14 +41,11 @@ namespace VulnCrawler
if (string.IsNullOrEmpty(cve)) {
continue;
}
string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}";
foreach (var parent in commit.Parents) {
try
{
// 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴
var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree);
......@@ -132,7 +129,7 @@ namespace VulnCrawler
};
/* VulnDB에 추가 */
VulnRDS._InsertVulnData(vuln);
//VulnRDS._InsertVulnData(vuln);
}
}
......
......@@ -11,7 +11,7 @@ using System.Threading.Tasks;
using VulnCrawler;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using LibGit2Sharp;
namespace VulnUserCodeAnalyzer
{
......@@ -42,7 +42,12 @@ namespace VulnUserCodeAnalyzer
public static void AutoLoad()
{
var dir = new DirectoryInfo(@"c:\CVE");
if (!dir.Exists)
{
Console.WriteLine("found not CVE Directory");
return;
}
Console.WriteLine("Loading CVE List...");
foreach (var json in dir.EnumerateFiles("*.json"))
{
var match = Regex.Match(json.Name, @"(20\d\d)");
......@@ -59,7 +64,7 @@ namespace VulnUserCodeAnalyzer
var dict = LoadCveJson(int.Parse(match.Value));
CveDict.Add(year, dict);
Console.WriteLine($"cve 로드 완료 {year}, 개수 : {CveDict[year].Count}");
Console.WriteLine($"Finished loading CVE List Year: {year}, Count : {CveDict[year].Count}");
}
}
......@@ -109,11 +114,50 @@ namespace VulnUserCodeAnalyzer
}
class Program
{
/// <summary>
/// Clone 콜백 함수
/// </summary>
/// <param name="progress"></param>
/// <returns></returns>
public static bool TransferProgress(TransferProgress progress)
{
int totalBytes = progress.TotalObjects;
int receivedBytes = progress.ReceivedObjects;
long receivedTotal = progress.ReceivedBytes;
double received = progress.ReceivedBytes / 1000000;
double percent = ((double)receivedBytes / (double)totalBytes);
Console.WriteLine($"Progress: {percent.ToString("P2")}, Remain: {receivedBytes} of {totalBytes}"); //, 받은 용량: {received.ToString()}MB");
Console.ForegroundColor = ConsoleColor.DarkGreen;
return true;
}
public static void CheckoutProcess(string path, int completedSteps, int totalSteps)
{
Console.WriteLine($"{completedSteps}, {totalSteps}, {path}");
}
public static void Clone(string path, string url)
{
Console.WriteLine($"Start Cloning Path : {path}");
string clone = Repository.Clone(url, $@"{path}", new CloneOptions { OnTransferProgress = TransferProgress, OnCheckoutProgress = CheckoutProcess });
Console.ResetColor();
Console.WriteLine($"Finished Clone Repository: {clone}");
}
static void Main(string[] args)
{
/* 연도별 CVE JSON 파일 로드 */
//CVE_JSON.AutoLoad();
CVE_JSON.AutoLoad();
/* 크롤러 타입 */
var crawler = new VulnC();
......@@ -129,7 +173,7 @@ namespace VulnUserCodeAnalyzer
AWS.LoadAccount(xml);
AWS.Account account = AWS.account;
/* AWS 정보 출력 */
Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}");
try
{
/* DB 접속 시도 */
......@@ -137,35 +181,71 @@ namespace VulnUserCodeAnalyzer
}
catch (Exception e)
{
Console.WriteLine($"접속 에러 :: {e.ToString()}");
Console.WriteLine($"Connection Error :: {e.ToString()}");
return;
}
/* AWS 연결 여부 확인 */
if (VulnRDS.Conn.State == System.Data.ConnectionState.Open)
{
Console.WriteLine("접속 성공");
Console.WriteLine("Connection Success");
}
else
{
Console.WriteLine("연결 실패");
Console.WriteLine("Fail Connection");
return;
}
while (true)
{
string userId = string.Empty;
string repoPath = string.Empty;
Stopwatch repoWatch = new Stopwatch();
repoWatch.Start();
while (true)
{
var elapsedSeconds = repoWatch.Elapsed.TotalSeconds;
if (elapsedSeconds < 10)
{
continue;
}
Console.WriteLine("Checking User DB...");
var reposits = VulnRDS.SelectAllReposit();
foreach (var (userName, repository) in reposits)
{
Console.WriteLine($"{userName}, {repository}");
if (string.IsNullOrWhiteSpace(repository))
{
continue;
}
var repoBytes = Encoding.Unicode.GetBytes(repository);
var repoBase64 = Convert.ToBase64String(repoBytes);
Console.ReadLine();
var repoDir = new DirectoryInfo($@"C:\Repo\{repoBase64}");
if (repoDir.Exists)
{
continue;
}
repoDir.Create();
Console.WriteLine($"Clone... Path : {repoDir.FullName}, Url : {repository}");
Clone(repoDir.FullName, repository);
repoPath = repoDir.FullName;
userId = userName;
}
if (!string.IsNullOrWhiteSpace(repoPath) && !string.IsNullOrWhiteSpace(userId))
{
break;
}
repoWatch.Restart();
}
//Console.WriteLine("엔터를 누르세요");
//Console.ReadLine();
/* hashDict = 사용된 사용자 함수 정보 */
var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
/* 경과 시간 체크 */
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
DirectoryInfo dirInfo = new DirectoryInfo(@"C:\code");
DirectoryInfo dirInfo = new DirectoryInfo(repoPath);
/* 모든 .c 파일 탐색 */
var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
......@@ -217,7 +297,7 @@ namespace VulnUserCodeAnalyzer
* CVE를 가지고 있다고 인정하는 프로그램 정책 때문
*/
var searchedCveHashList = VulnRDS.SelectVulnbyCve(cve);
Console.WriteLine($"cve:{cve}, {searchedCveHashList.Count()}개 가져옴");
Console.WriteLine($"CVE:{cve}, Received Count : {searchedCveHashList.Count()}");
foreach (var s in searchedCveHashList)
{
vulnHashSet.Add(s);
......@@ -231,7 +311,7 @@ namespace VulnUserCodeAnalyzer
/* 본격적인 취약점 매칭 부분 */
foreach (var vulnSet in vulnDict)
{
//Console.WriteLine($"-----cve:{vulnSet.Key}");
Console.WriteLine($"-----cve:{vulnSet.Key}");
bool match = false;
foreach (var vuln in vulnSet.Value)
{
......@@ -243,6 +323,7 @@ namespace VulnUserCodeAnalyzer
{
if (hashDict.ContainsKey(vuln.LenFunc))
{
//Console.WriteLine("찾음");
/* Bloom Filter는 아쉽게도 포함 여부만 알 수 있기에
* 포함되었음을 알았다면 검색해서 정보를 구한다. */
var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
......@@ -271,13 +352,13 @@ namespace VulnUserCodeAnalyzer
/* 취약점 레코드가 전부 있어야 CVE 찾음 인정 */
if (match)
{
Console.WriteLine($"CVE 찾음 {vulnSet.Key}");
Console.WriteLine($"Matched CVE : {vulnSet.Key}");
/* 찾았으면 cve값을 기록함 밑에서 찾은 cve 정보 전송하기 위해 */
findCveList.Add(vulnSet.Key);
}
else
{
Console.WriteLine("없음");
Console.WriteLine("Not");
}
}
stopwatch.Stop();
......@@ -285,8 +366,10 @@ namespace VulnUserCodeAnalyzer
var hours = stopwatch.Elapsed.Hours;
var minutes = stopwatch.Elapsed.Minutes;
var seconds = stopwatch.Elapsed.Seconds;
Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
Console.WriteLine($"찾은 CVE 개수 : {findCveList.Count}");
Console.WriteLine($"Elapsed Time : {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
Console.WriteLine($"Matched CVE Count : {findCveList.Count}");
//Console.ReadLine();
var yearMatch = new Regex(@"CVE-(\d{4})-(\d+)");
foreach (var cve in findCveList)
{
......@@ -340,9 +423,9 @@ namespace VulnUserCodeAnalyzer
var urlBytes = Convert.FromBase64String(findCveDict[cve].FirstOrDefault().Url);
string url = Encoding.Unicode.GetString(urlBytes);
//Console.WriteLine(findCveDict[cve].FirstOrDefault().Path.Replace(repoPath, ""));
/* DB 전송 */
VulnRDS.InsertVulnDetail(new VulnRDS.Vuln_detail
var vulnDetail = new VulnRDS.Vuln_detail
{
CveName = data.Code,
Type = type,
......@@ -351,13 +434,18 @@ namespace VulnUserCodeAnalyzer
CveDetail = data.Detail,
Publish_date = data.Publish_Date.ToString("yyyy-MM-dd"),
Update_date = data.Update_Date.ToString("yyyy-MM-dd"),
UserName = "samsung",
UserName = userId,
Url = url,
FileName = findCveDict[cve].FirstOrDefault().Path.Replace(@"C:\code", ""),
FileName = findCveDict[cve].FirstOrDefault().Path.Replace(repoPath, ""),
FuncName = findCveDict[cve].FirstOrDefault().FuncName,
Product = data.Type,
});
Console.WriteLine("추가 완료");
};
/* DB 전송 */
VulnRDS.InsertVulnDetail(vulnDetail);
Console.WriteLine($"Added CVE: {vulnDetail.CveName}, Type: {vulnDetail.Type}, CVSS: {vulnDetail.Level}");
}
}
}
}
......
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props" Condition="Exists('..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props')" />
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
......@@ -11,6 +12,8 @@
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
......@@ -32,6 +35,9 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="LibGit2Sharp, Version=0.25.0.0, Culture=neutral, PublicKeyToken=7cbde695407f0333, processorArchitecture=MSIL">
<HintPath>..\packages\LibGit2Sharp.0.25.2\lib\netstandard2.0\LibGit2Sharp.dll</HintPath>
</Reference>
<Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" />
<Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
......@@ -64,4 +70,10 @@
</ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>이 프로젝트는 이 컴퓨터에 없는 NuGet 패키지를 참조합니다. 해당 패키지를 다운로드하려면 NuGet 패키지 복원을 사용하십시오. 자세한 내용은 http://go.microsoft.com/fwlink/?LinkID=322105를 참조하십시오. 누락된 파일은 {0}입니다.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\LibGit2Sharp.NativeBinaries.1.0.217\build\net461\LibGit2Sharp.NativeBinaries.props'))" />
</Target>
</Project>
\ No newline at end of file
......
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="LibGit2Sharp" version="0.25.2" targetFramework="net461" />
<package id="LibGit2Sharp.NativeBinaries" version="1.0.217" targetFramework="net461" />
<package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" />
<package id="System.ValueTuple" version="4.5.0" targetFramework="net461" />
</packages>
\ No newline at end of file
......