노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
노현종
c6c1c609 1 parent 1ef87191

1차 완성(테스트 완료)

Expand all
Inline Side-by-side
Showing 8 changed files with 201 additions and 75 deletions
......@@ -21,10 +21,7 @@ namespace VulnCrawler
#region MySql 연결
//SecureString s_key = GetConsoleSecurePassword();
//Console.Clear();
//string key = SecureStringToString(s_key);
//AWS.account.Id = "yhackerbv";
//AWS.account.Pw = "guswhd12";
//AWS.account.Endpoint = "vulndb.cby38wfppa7l.us-east-2.rds.amazonaws.com";
//string key = SecureStringToString(s_key)
//AWS.SaveAccount();
//AES aes = new AES();
......@@ -81,7 +78,7 @@ namespace VulnCrawler
/* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */
if (directory.Contains("linux"))
{
continue;
// continue;
}
// 템플릿 패턴화 T : VulnAbstractCrawler
VulnWorker.Run<VulnC>(directory);
......
......@@ -65,8 +65,12 @@ namespace VulnCrawler
}
// 소멸자
~VulnAbstractCrawler() {
try
{
Repository?.Dispose();
}
catch { }
}
private void LoadReservedList()
{
......@@ -223,7 +227,7 @@ namespace VulnCrawler
// 괄호가 모두 닫혔으니 종료
if (bracketCount < 0)
{
Console.WriteLine("괄호끝");
// Console.WriteLine("괄호끝");
break;
}
// oldBuilder.AppendLine(line);
......@@ -271,34 +275,36 @@ namespace VulnCrawler
string md5 = string.Empty;
if (item.Value.Count() != 0)
{
Console.WriteLine("크리티컬 변수 목록");
Console.ForegroundColor = ConsoleColor.Cyan;
foreach (var c in item.Value)
{
Console.WriteLine(c);
}
Console.ResetColor();
Console.WriteLine("-------------------");
//Console.WriteLine("크리티컬 변수 목록");
//Console.ForegroundColor = ConsoleColor.Cyan;
//foreach (var c in item.Value)
//{
// Console.WriteLine(c);
//}
//Console.ResetColor();
//Console.WriteLine("-------------------");
// 크리티컬 블록 추출
var blocks = GetCriticalBlocks(func, item.Value).ToList();
if (blocks == null)
{
continue;
}
foreach (var block in blocks)
{
var blocks = new List<Block>();
//var blocks = GetCriticalBlocks(func, item.Value).ToList();
//if (blocks == null)
//{
// continue;
//}
//foreach (var block in blocks)
//{
block.CriticalList = item.Value;
/* 추상화 및 정규화 */
block.AbsCode = Abstract(block.Code, varTable, methodTable);
block.Hash = MD5HashFunc(block.AbsCode);
// block.CriticalList = item.Value;
// /* 추상화 및 정규화 */
// block.AbsCode = Abstract(block.Code, varTable, methodTable);
// block.Hash = MD5HashFunc(block.AbsCode);
}
//}
/* 추상화 변환 테이블 출력 */
foreach (var var in varTable)
{
Console.WriteLine($"{var.Key}, {var.Value}");
}
//foreach (var var in varTable)
//{
// Console.WriteLine($"{var.Key}, {var.Value}");
//}
yield return (methodName, func, blocks);
}
......@@ -644,7 +650,7 @@ namespace VulnCrawler
/// </summary>
/// <param name="str">INPUT 문자열</param>
/// <returns>결과 문자열</returns>
protected static string MD5HashFunc(string str) {
public static string MD5HashFunc(string str) {
StringBuilder MD5Str = new StringBuilder();
byte[] byteArr = Encoding.ASCII.GetBytes(str);
byte[] resultArr = (new MD5CryptoServiceProvider()).ComputeHash(byteArr);
......
......@@ -26,7 +26,8 @@ namespace VulnCrawler
/// <param name="patchCode">패치 코드</param>
/// <returns></returns>
public override MatchCollection GetMatches(string patchCode) {
var regs = Regex.Matches(patchCode, RegexFuncPattern);
var funcPattern = $@"(?<{MethodName}>(unsigned|static)?( const )? [\w]+ [\w]+\(([\w \*\,\t\n])*[\)\,])";
var regs = Regex.Matches(patchCode, funcPattern);
return regs;
}
/// <summary>
......@@ -82,7 +83,7 @@ namespace VulnCrawler
{
continue;
}
Console.WriteLine(line);
// Console.WriteLine(line);
builder.AppendLine(line);
continue;
}
......
......@@ -139,7 +139,9 @@ namespace VulnCrawler
//콘솔출력용
sql = "INSERT INTO vuln_Info(vulnId, cve, funcName, lenFunc, code, blockHash, url) " +
$"VALUES({last_vulnId}, {vuln.Cve}, '{vuln.FuncName}', '{vuln.LenFunc}', {vuln.Code},'{vuln.BlockHash}', '{vuln.Url}')";
Console.WriteLine(sql);
//Console.WriteLine(sql);
//Console.ReadLine();
}
catch (Exception e)
{
......@@ -150,7 +152,7 @@ namespace VulnCrawler
Connect(Account, DbName);
goto Retry;
}
Console.ReadLine();
//Console.ReadLine();
}
}
public static void InsertUserData(User user)
......
......@@ -20,15 +20,26 @@ namespace VulnCrawler
crawler.Init(dirPath);
/* 초기화된 커밋 목록 가져옴 */
var commits = crawler.Commits;
int totalCount = commits.Count();
int count = 0;
foreach (var commit in commits) {
// 커밋 메시지
count++;
double per = ((double)count / (double)totalCount) * 100;
Console.Clear();
Console.WriteLine($"{count} / {totalCount} :: {per.ToString("#0.0")}%");
string message = commit.Message;
string cve = crawler.GetCVE(message);
if (string.IsNullOrEmpty(cve)) {
continue;
}
foreach (var parent in commit.Parents) {
try
{
// 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴
var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree);
// 패치 엔트리 파일 배열 중에 파일 확장자가 .py인 것만 가져옴
......@@ -41,6 +52,9 @@ namespace VulnCrawler
PrintPatchEntrys(entrys, crawler, message, cve, repoName);
// Console.ReadLine();
}
catch(Exception)
{ }
}
}
}
......@@ -64,33 +78,33 @@ namespace VulnCrawler
// 출력
if (regs.Count > 0)
{
int deleted = entry.LinesDeleted;
if (deleted == 0)
{
continue;
}
Console.BackgroundColor = ConsoleColor.DarkBlue;
Console.WriteLine($"Old Content: \n{oldContent}");
Console.ResetColor();
Console.ForegroundColor = ConsoleColor.Blue;
Console.WriteLine($"status: {entry.Status.ToString()}");
Console.WriteLine($"added: {entry.LinesAdded.ToString()}, deleted: {entry.LinesDeleted.ToString()}");
Console.WriteLine($"old path: {entry.OldPath.ToString()}, new path: {entry.Path.ToString()}");
Console.ResetColor();
Console.Write($"CVE: ");
Console.ForegroundColor = ConsoleColor.Red;
Console.Write($"{cve}");
Console.WriteLine("");
Console.ResetColor();
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine($"Commit Message: {commitMsg}");
Console.ResetColor();
Console.BackgroundColor = ConsoleColor.DarkRed;
Console.WriteLine($"Patched: \n{entry.Patch}");
Console.ResetColor();
//int deleted = entry.LinesDeleted;
//if (deleted == 0)
//{
// // continue;
//}
//Console.BackgroundColor = ConsoleColor.DarkBlue;
//Console.WriteLine($"Old Content: \n{oldContent}");
//Console.ResetColor();
//Console.ForegroundColor = ConsoleColor.Blue;
//Console.WriteLine($"status: {entry.Status.ToString()}");
//Console.WriteLine($"added: {entry.LinesAdded.ToString()}, deleted: {entry.LinesDeleted.ToString()}");
//Console.WriteLine($"old path: {entry.OldPath.ToString()}, new path: {entry.Path.ToString()}");
//Console.ResetColor();
//Console.Write($"CVE: ");
//Console.ForegroundColor = ConsoleColor.Red;
//Console.Write($"{cve}");
//Console.WriteLine("");
//Console.ResetColor();
//Console.ForegroundColor = ConsoleColor.Yellow;
//Console.WriteLine($"Commit Message: {commitMsg}");
//Console.ResetColor();
//Console.BackgroundColor = ConsoleColor.DarkRed;
//Console.WriteLine($"Patched: \n{entry.Patch}");
//Console.ResetColor();
/* 패치된 코드들에서 Method로 나누고 크리티컬 변수로 뽑아옴 Dictionary 구조 (키 = 함수명) */
var table = self.ExtractGitCriticalMethodTable(entry.Patch);
/* 크리티컬 메서드 테이블과 패치 전 파일에서 Process 하고 tuple로 가져옴 */
......@@ -98,10 +112,10 @@ namespace VulnCrawler
{
/* 메서드 이름, 원본 함수 코드, 블록 리스트(크리티컬 포함) */
(var methodName, var oriFunc, var blocks) = tuple;
Console.BackgroundColor = ConsoleColor.DarkRed;
Console.WriteLine($"메서드 이름 : {methodName}");
Console.ResetColor();
//foreach (var block in blocks)
//Console.BackgroundColor = ConsoleColor.DarkRed;
//Console.WriteLine($"메서드 이름 : {methodName}");
//Console.ResetColor();
////foreach (var block in blocks)
//{
// /* 크리티컬 블록이 아니면 볼 필요 없으니 넘어감 */
// if (!block.HasCritical)
......@@ -146,26 +160,41 @@ namespace VulnCrawler
/* VulnDB에 추가 */
//VulnRDS.InsertVulnData(vuln);
//}
if (string.IsNullOrWhiteSpace(oriFunc))
{
continue;
}
string abstractCode = self.Abstract(oriFunc, new Dictionary<string, string>(), new Dictionary<string, string>());
byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName);
byte[] absCodeBytes = Encoding.Unicode.GetBytes(abstractCode);
VulnRDS.Vuln vuln = new VulnRDS.Vuln()
byte[] commitMsgBytes = Encoding.Unicode.GetBytes(commitMsg);
byte[] funcBytes = Encoding.Unicode.GetBytes(oriFunc);
VulnRDS._Vuln vuln = new VulnRDS._Vuln()
{
BlockHash = Convert.ToBase64String(absCodeBytes),
LenFunc = oriFunc.Length,
Cve = cve,
LenBlock = oriFunc.Length,
BlockHash = VulnAbstractCrawler.MD5HashFunc(Convert.ToBase64String(absCodeBytes)),
FuncName = Convert.ToBase64String(funcNameBytes),
Code = Convert.ToBase64String(funcBytes),
Url = Convert.ToBase64String(commitMsgBytes),
//BlockHash = Convert.ToBase64String(absCodeBytes),
//Cve = cve,
//LenBlock = oriFunc.Length,
//FuncName = Convert.ToBase64String(funcNameBytes),
};
Console.WriteLine(vuln.BlockHash);
Console.ReadLine();
// Console.WriteLine(vuln.BlockHash);
// Console.ReadLine();
/* VulnDB에 추가 */
//VulnRDS.InsertVulnData(vuln);
VulnRDS._InsertVulnData(vuln);
}
}
else
{
//Console.WriteLine("zzz");
//Console.ReadLine();
continue;
}
......@@ -174,6 +203,8 @@ namespace VulnCrawler
}
catch (Exception e)
{
//Console.WriteLine(e.ToString());
//Console.ReadLine();
continue;
}
......
This diff is collapsed. Click to expand it.

using BloomFilter;
using System;
using System.Collections.Generic;
using System.IO;
......@@ -14,6 +15,47 @@ namespace VulnUserCodeAnalyzer
{
static void Main(string[] args)
{
// default usage
int capacity = 20000000;
var filter = new Filter<string>(capacity);
//filter.Add("1");
// filter.Add("1");
//Console.WriteLine(filter.Contains("1"));
//Console.WriteLine(filter.Contains("content2"));
/* AWS 계정 정보 파일 읽음 */
string txt = File.ReadAllText(@"Account.xml");
// string xml = aes.AESDecrypt128(txt, key);
string xml = txt;
AWS.LoadAccount(xml);
AWS.Account account = AWS.account;
/* AWS 정보 출력 */
Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
try
{
/* DB 접속 시도 */
VulnRDS.Connect(account, "vuln");
}
catch (Exception e)
{
Console.WriteLine($"접속 에러 :: {e.ToString()}");
}
/* AWS 연결 여부 확인 */
if (VulnRDS.Conn.State == System.Data.ConnectionState.Open)
{
Console.WriteLine("접속 성공");
}
else
{
Console.WriteLine("연결 실패");
return;
}
var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code");
......@@ -39,6 +81,7 @@ namespace VulnUserCodeAnalyzer
{
hash.Path = codeFile.FullName;
hashDict[item.Key].Add(hash);
filter.Add(hash.Hash);
}
}
......@@ -48,7 +91,7 @@ namespace VulnUserCodeAnalyzer
Console.Clear();
Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}");
//if (count > 20)
//if (count > 100)
//{
// break;
//}
......@@ -60,12 +103,56 @@ namespace VulnUserCodeAnalyzer
foreach (var set in hashDict)
{
Console.WriteLine($"-----key:{set.Key}");
foreach (var hash in set.Value)
var vulnList = VulnRDS.SelectVulnbyLen(set.Key);
foreach (var vuln in vulnList)
{
// Console.WriteLine(vuln.BlockHash);
if (filter.Contains(vuln.BlockHash))
{
Console.WriteLine($"필터 확인 : {vuln.BlockHash}");
if (hashDict.ContainsKey(vuln.LenFunc))
{
Console.WriteLine($"{hash.FuncName}, {hash.Hash}, {hash.Len}, {hash.Path}");
var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
if (userBlock == null)
{
Console.WriteLine("userBlock이 비어있습니다.");
continue;
}
Console.WriteLine($"{userBlock.FuncName} 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}");
}
}
}
//foreach (var hash in set.Value)
//{
// Console.WriteLine($"{hash.FuncName}, {hash.Hash}, {hash.Len}, {hash.Path}");
//}
}
// 블룸 필터 테스트
//while(true)
//{
// string key = Console.ReadLine();
// if (key == "-1")
// {
// break;
// }
// if (filter.Contains(key))
// {
// Console.WriteLine("포함");
// }
// else
// {
// Console.WriteLine("없음");
// }
//}
}
......
......@@ -32,6 +32,7 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" />
<Reference Include="System" />
<Reference Include="System.Core" />
<Reference Include="System.Xml.Linq" />
......@@ -42,6 +43,7 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="BloomFilter.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
</ItemGroup>
......