노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
노현종
40478042 1 parent c6c1c609

매칭, DB 등 버그 수정

Inline Side-by-side
Showing 5 changed files with 84 additions and 110 deletions
......@@ -12,6 +12,7 @@ using System.Threading.Tasks;
using AESENC;
using System.Security;
using System.Runtime.InteropServices;
using System.Diagnostics;
namespace VulnCrawler
{
......@@ -73,6 +74,9 @@ namespace VulnCrawler
Console.WriteLine("Repository 목록 찾기 실패");
return;
}
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// Repository 목록 만큼 반복함.
foreach (var directory in directorys) {
/* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */
......@@ -83,6 +87,15 @@ namespace VulnCrawler
// 템플릿 패턴화 T : VulnAbstractCrawler
VulnWorker.Run<VulnC>(directory);
}
stopwatch.Stop();
var hours = stopwatch.Elapsed.TotalHours;
var minutes = stopwatch.Elapsed.TotalMinutes;
var seconds = stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
}
#region Secure string input
static String SecureStringToString(SecureString value) {
......
......@@ -46,6 +46,7 @@ namespace VulnCrawler
return hashCode;
}
}
public string PushUrl { get; set; }
protected Regex extractMethodLine;
protected HashSet<string> ReservedList { get; }
protected abstract string ReservedFileName { get; }
......@@ -106,7 +107,14 @@ namespace VulnCrawler
Console.WriteLine("로딩중");
Console.WriteLine(path);
Repository = new Repository(path);
PushUrl = Repository.Network.Remotes.FirstOrDefault().PushUrl;
if (PushUrl.EndsWith(".git"))
{
PushUrl = PushUrl.Replace(".git", "");
}
Console.WriteLine("로딩 완료");
Commits = SearchCommits();
Console.WriteLine($"Commits Count: {Commits.Count()}");
......
......@@ -716,11 +716,13 @@ namespace VulnCrawler
{
dict[obStr.Length] = new HashSet<UserBlock>();
}
byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray());
(dict[obStr.Length] as HashSet<UserBlock>).Add(new UserBlock
{
Hash = MD5HashFunc(obStr),
Hash = MD5HashFunc(Convert.ToBase64String(obStrBytes)),
Len = obStr.Length,
FuncName = funcName,
});
......@@ -859,12 +861,14 @@ namespace VulnCrawler
{
dict[obStr.Length] = new HashSet<UserBlock>();
}
byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray());
(dict[obStr.Length] as HashSet<UserBlock>).Add(new UserBlock
{
Hash = MD5HashFunc(obStr),
Hash = MD5HashFunc(Convert.ToBase64String(obStrBytes)),
Len = obStr.Length,
FuncName = funcName,
......
......@@ -22,20 +22,29 @@ namespace VulnCrawler
var commits = crawler.Commits;
int totalCount = commits.Count();
int count = 0;
string dir = Path.Combine(dirPath, "url.txt");
if (File.Exists(dir))
{
crawler.PushUrl = File.ReadAllText(dir);
}
foreach (var commit in commits) {
// 커밋 메시지
count++;
double per = ((double)count / (double)totalCount) * 100;
Console.Clear();
Console.WriteLine($"{count} / {totalCount} :: {per.ToString("#0.0")}%");
string message = commit.Message;
string cve = crawler.GetCVE(message);
if (string.IsNullOrEmpty(cve)) {
continue;
}
string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}";
foreach (var parent in commit.Parents) {
try
......@@ -49,7 +58,7 @@ namespace VulnCrawler
var dsp = dirPath.Split(Path.DirectorySeparatorChar);
string repoName = dsp[dsp.Length - 1];
// 현재 커밋에 대한 패치 엔트리 배열을 출력함
PrintPatchEntrys(entrys, crawler, message, cve, repoName);
PrintPatchEntrys(entrys, crawler, message, cve, repoName, commitUrl);
// Console.ReadLine();
}
catch(Exception)
......@@ -58,7 +67,7 @@ namespace VulnCrawler
}
}
private static void PrintPatchEntrys(IEnumerable<PatchEntryChanges> entrys, VulnAbstractCrawler self, string commitMsg, string cve, string repoName) {
private static void PrintPatchEntrys(IEnumerable<PatchEntryChanges> entrys, VulnAbstractCrawler self, string commitMsg, string cve, string repoName, string commitUrl) {
foreach (var entry in entrys) {
// 기존 소스코드
var oldOid = entry.OldOid;
......@@ -78,33 +87,6 @@ namespace VulnCrawler
// 출력
if (regs.Count > 0)
{
//int deleted = entry.LinesDeleted;
//if (deleted == 0)
//{
// // continue;
//}
//Console.BackgroundColor = ConsoleColor.DarkBlue;
//Console.WriteLine($"Old Content: \n{oldContent}");
//Console.ResetColor();
//Console.ForegroundColor = ConsoleColor.Blue;
//Console.WriteLine($"status: {entry.Status.ToString()}");
//Console.WriteLine($"added: {entry.LinesAdded.ToString()}, deleted: {entry.LinesDeleted.ToString()}");
//Console.WriteLine($"old path: {entry.OldPath.ToString()}, new path: {entry.Path.ToString()}");
//Console.ResetColor();
//Console.Write($"CVE: ");
//Console.ForegroundColor = ConsoleColor.Red;
//Console.Write($"{cve}");
//Console.WriteLine("");
//Console.ResetColor();
//Console.ForegroundColor = ConsoleColor.Yellow;
//Console.WriteLine($"Commit Message: {commitMsg}");
//Console.ResetColor();
//Console.BackgroundColor = ConsoleColor.DarkRed;
//Console.WriteLine($"Patched: \n{entry.Patch}");
//Console.ResetColor();
/* 패치된 코드들에서 Method로 나누고 크리티컬 변수로 뽑아옴 Dictionary 구조 (키 = 함수명) */
var table = self.ExtractGitCriticalMethodTable(entry.Patch);
/* 크리티컬 메서드 테이블과 패치 전 파일에서 Process 하고 tuple로 가져옴 */
......@@ -112,80 +94,32 @@ namespace VulnCrawler
{
/* 메서드 이름, 원본 함수 코드, 블록 리스트(크리티컬 포함) */
(var methodName, var oriFunc, var blocks) = tuple;
//Console.BackgroundColor = ConsoleColor.DarkRed;
//Console.WriteLine($"메서드 이름 : {methodName}");
//Console.ResetColor();
////foreach (var block in blocks)
//{
// /* 크리티컬 블록이 아니면 볼 필요 없으니 넘어감 */
// if (!block.HasCritical)
// {
// // Console.WriteLine("크리티컬 아님");
// continue;
// }
// if (block.HasCritical)
// {
// Console.BackgroundColor = ConsoleColor.DarkMagenta;
// }
// else
// {
// Console.BackgroundColor = ConsoleColor.DarkGreen;
// }
// /* 블록 정보 출력(블록 번호, 블록 소스코드, 블록 추상화 코드, 블록 해쉬값) */
// Console.WriteLine($"=====block({block.Num}, {block.HasCritical.ToString()})");
// Console.WriteLine(block.Code);
// Console.ResetColor();
// Console.WriteLine($"AbsCode = \n{block.AbsCode}");
// Console.WriteLine($"MD5 = {block.Hash}");
// /* base64 인코딩(MySQL에 들어갈 수 없는 문자열이 있을 수 있으므로 인코딩) */
// byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName);
// byte[] codeOriBeforeBytes = Encoding.Unicode.GetBytes(oriFunc);
// byte[] codeAbsBeforeBytes = Encoding.Unicode.GetBytes(block.AbsCode);
// /* VulnDB에 하나의 레코드로 들어가는 하나의 취약점 객체 */
// VulnRDS.Vuln vuln = new VulnRDS.Vuln()
// {
// Cve = cve,
// BlockHash = block.Hash,
// LenBlock = block.Code.Length,
// FuncName = Convert.ToBase64String(funcNameBytes),
// //CodeOriBefore = Convert.ToBase64String(codeOriBeforeBytes),
// //CodeAbsBefore = Convert.ToBase64String(codeAbsBeforeBytes),
// //NumBlock = block.Num,
// };
// Console.WriteLine($"Vuln FuncName:{vuln.FuncName}");
/* VulnDB에 추가 */
//VulnRDS.InsertVulnData(vuln);
//}
if (string.IsNullOrWhiteSpace(oriFunc))
{
continue;
}
string abstractCode = self.Abstract(oriFunc, new Dictionary<string, string>(), new Dictionary<string, string>());
byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName);
byte[] absCodeBytes = Encoding.Unicode.GetBytes(abstractCode);
byte[] commitMsgBytes = Encoding.Unicode.GetBytes(commitMsg);
byte[] commitUrlBytes = Encoding.Unicode.GetBytes(commitUrl);
byte[] funcBytes = Encoding.Unicode.GetBytes(oriFunc);
string absCodeBase64 = Convert.ToBase64String(absCodeBytes);
VulnRDS._Vuln vuln = new VulnRDS._Vuln()
{
LenFunc = oriFunc.Length,
LenFunc = absCodeBase64.Length,
Cve = cve,
BlockHash = VulnAbstractCrawler.MD5HashFunc(Convert.ToBase64String(absCodeBytes)),
BlockHash = VulnAbstractCrawler.MD5HashFunc(absCodeBase64),
FuncName = Convert.ToBase64String(funcNameBytes),
Code = Convert.ToBase64String(funcBytes),
Url = Convert.ToBase64String(commitMsgBytes),
//BlockHash = Convert.ToBase64String(absCodeBytes),
//Cve = cve,
//LenBlock = oriFunc.Length,
//FuncName = Convert.ToBase64String(funcNameBytes),
Url = Convert.ToBase64String(commitUrlBytes),
};
// Console.WriteLine(vuln.BlockHash);
// Console.ReadLine();
/* VulnDB에 추가 */
VulnRDS._InsertVulnData(vuln);
......@@ -193,8 +127,6 @@ namespace VulnCrawler
}
else
{
//Console.WriteLine("zzz");
//Console.ReadLine();
continue;
}
......@@ -203,8 +135,6 @@ namespace VulnCrawler
}
catch (Exception e)
{
//Console.WriteLine(e.ToString());
//Console.ReadLine();
continue;
}
......
......@@ -2,6 +2,7 @@
using BloomFilter;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
......@@ -15,14 +16,17 @@ namespace VulnUserCodeAnalyzer
{
static void Main(string[] args)
{
var crawler = new VulnC();
//var bytes = Convert.FromBase64String("dgBvAGkAZAAgAGsAdgBtAF8AbQBtAHUAXwBuAGUAdwBfAGMAcgAzACgAcwB0AHIAdQBjAHQAIABrAHYAbQBfAHYAYwBwAHUAIAAqAHYAYwBwAHUAKQANAAoAewANAAoACQBtAG0AdQBfAGYAcgBlAGUAXwByAG8AbwB0AHMAKAB2AGMAcAB1ACkAOwANAAoAfQANAAoA");
//var str = Encoding.Unicode.GetString(bytes);
//Console.WriteLine(str);
//Console.WriteLine(crawler.Abstract(str, new Dictionary<string, string>(), new Dictionary<string, string>()));
//Console.ReadLine();
// default usage
int capacity = 20000000;
int capacity = 50000000;
var filter = new Filter<string>(capacity);
//filter.Add("1");
// filter.Add("1");
//Console.WriteLine(filter.Contains("1"));
//Console.WriteLine(filter.Contains("content2"));
/* AWS 계정 정보 파일 읽음 */
string txt = File.ReadAllText(@"Account.xml");
......@@ -42,6 +46,7 @@ namespace VulnUserCodeAnalyzer
catch (Exception e)
{
Console.WriteLine($"접속 에러 :: {e.ToString()}");
return;
}
/* AWS 연결 여부 확인 */
......@@ -58,13 +63,16 @@ namespace VulnUserCodeAnalyzer
var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code");
var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
int totalFileCount = codeFiles.Count();
var crawler = new VulnC();
int count = 0;
foreach (var codeFile in codeFiles)
{
Console.WriteLine(codeFile.FullName);
using (var reader = codeFile.OpenText())
{
......@@ -91,14 +99,16 @@ namespace VulnUserCodeAnalyzer
Console.Clear();
Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}");
//if (count > 100)
//{
// break;
//}
if (count > 100)
{
break;
}
}
}
var findBlocks = new Queue<VulnAbstractCrawler.UserBlock>();
foreach (var set in hashDict)
{
......@@ -118,10 +128,8 @@ namespace VulnUserCodeAnalyzer
Console.WriteLine("userBlock이 비어있습니다.");
continue;
}
Console.WriteLine($"{userBlock.FuncName} 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}");
Console.WriteLine($"CVE:{vuln.Cve}, {userBlock.FuncName}, 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}");
findBlocks.Enqueue(userBlock);
}
}
......@@ -133,6 +141,17 @@ namespace VulnUserCodeAnalyzer
//}
}
stopwatch.Stop();
var hours = stopwatch.Elapsed.TotalHours;
var minutes = stopwatch.Elapsed.TotalMinutes;
var seconds = stopwatch.Elapsed.TotalSeconds;
Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
// 블룸 필터 테스트
//while(true)
......