노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
노현종
40478042 1 parent c6c1c609

매칭, DB 등 버그 수정

Inline Side-by-side
Showing 5 changed files with 84 additions and 110 deletions
...@@ -12,6 +12,7 @@ using System.Threading.Tasks; ...@@ -12,6 +12,7 @@ using System.Threading.Tasks;
12 using AESENC; 12 using AESENC;
13 using System.Security; 13 using System.Security;
14 using System.Runtime.InteropServices; 14 using System.Runtime.InteropServices;
15 +using System.Diagnostics;
15 16
16 namespace VulnCrawler 17 namespace VulnCrawler
17 { 18 {
...@@ -73,6 +74,9 @@ namespace VulnCrawler ...@@ -73,6 +74,9 @@ namespace VulnCrawler
73 Console.WriteLine("Repository 목록 찾기 실패"); 74 Console.WriteLine("Repository 목록 찾기 실패");
74 return; 75 return;
75 } 76 }
77 +
78 + Stopwatch stopwatch = new Stopwatch();
79 + stopwatch.Start();
76 // Repository 목록 만큼 반복함. 80 // Repository 목록 만큼 반복함.
77 foreach (var directory in directorys) { 81 foreach (var directory in directorys) {
78 /* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */ 82 /* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */
...@@ -83,6 +87,15 @@ namespace VulnCrawler ...@@ -83,6 +87,15 @@ namespace VulnCrawler
83 // 템플릿 패턴화 T : VulnAbstractCrawler 87 // 템플릿 패턴화 T : VulnAbstractCrawler
84 VulnWorker.Run<VulnC>(directory); 88 VulnWorker.Run<VulnC>(directory);
85 } 89 }
90 + stopwatch.Stop();
91 + var hours = stopwatch.Elapsed.TotalHours;
92 + var minutes = stopwatch.Elapsed.TotalMinutes;
93 + var seconds = stopwatch.Elapsed.TotalSeconds;
94 +
95 + Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
96 +
97 +
98 +
86 } 99 }
87 #region Secure string input 100 #region Secure string input
88 static String SecureStringToString(SecureString value) { 101 static String SecureStringToString(SecureString value) {
......
...@@ -46,6 +46,7 @@ namespace VulnCrawler ...@@ -46,6 +46,7 @@ namespace VulnCrawler
46 return hashCode; 46 return hashCode;
47 } 47 }
48 } 48 }
49 + public string PushUrl { get; set; }
49 protected Regex extractMethodLine; 50 protected Regex extractMethodLine;
50 protected HashSet<string> ReservedList { get; } 51 protected HashSet<string> ReservedList { get; }
51 protected abstract string ReservedFileName { get; } 52 protected abstract string ReservedFileName { get; }
...@@ -106,7 +107,14 @@ namespace VulnCrawler ...@@ -106,7 +107,14 @@ namespace VulnCrawler
106 Console.WriteLine("로딩중"); 107 Console.WriteLine("로딩중");
107 Console.WriteLine(path); 108 Console.WriteLine(path);
108 Repository = new Repository(path); 109 Repository = new Repository(path);
109 - 110 + PushUrl = Repository.Network.Remotes.FirstOrDefault().PushUrl;
111 +
112 + if (PushUrl.EndsWith(".git"))
113 + {
114 + PushUrl = PushUrl.Replace(".git", "");
115 +
116 + }
117 +
110 Console.WriteLine("로딩 완료"); 118 Console.WriteLine("로딩 완료");
111 Commits = SearchCommits(); 119 Commits = SearchCommits();
112 Console.WriteLine($"Commits Count: {Commits.Count()}"); 120 Console.WriteLine($"Commits Count: {Commits.Count()}");
......
...@@ -716,11 +716,13 @@ namespace VulnCrawler ...@@ -716,11 +716,13 @@ namespace VulnCrawler
716 { 716 {
717 dict[obStr.Length] = new HashSet<UserBlock>(); 717 dict[obStr.Length] = new HashSet<UserBlock>();
718 } 718 }
719 + byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
720 +
719 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray()); 721 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray());
720 722
721 (dict[obStr.Length] as HashSet<UserBlock>).Add(new UserBlock 723 (dict[obStr.Length] as HashSet<UserBlock>).Add(new UserBlock
722 { 724 {
723 - Hash = MD5HashFunc(obStr), 725 + Hash = MD5HashFunc(Convert.ToBase64String(obStrBytes)),
724 Len = obStr.Length, 726 Len = obStr.Length,
725 FuncName = funcName, 727 FuncName = funcName,
726 }); 728 });
...@@ -859,12 +861,14 @@ namespace VulnCrawler ...@@ -859,12 +861,14 @@ namespace VulnCrawler
859 { 861 {
860 dict[obStr.Length] = new HashSet<UserBlock>(); 862 dict[obStr.Length] = new HashSet<UserBlock>();
861 } 863 }
864 + byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
865 +
862 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray()); 866 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray());
863 867
864 868
865 (dict[obStr.Length] as HashSet<UserBlock>).Add(new UserBlock 869 (dict[obStr.Length] as HashSet<UserBlock>).Add(new UserBlock
866 { 870 {
867 - Hash = MD5HashFunc(obStr), 871 + Hash = MD5HashFunc(Convert.ToBase64String(obStrBytes)),
868 Len = obStr.Length, 872 Len = obStr.Length,
869 FuncName = funcName, 873 FuncName = funcName,
870 874
......
...@@ -22,20 +22,29 @@ namespace VulnCrawler ...@@ -22,20 +22,29 @@ namespace VulnCrawler
22 var commits = crawler.Commits; 22 var commits = crawler.Commits;
23 int totalCount = commits.Count(); 23 int totalCount = commits.Count();
24 int count = 0; 24 int count = 0;
25 + string dir = Path.Combine(dirPath, "url.txt");
26 +
27 + if (File.Exists(dir))
28 + {
29 + crawler.PushUrl = File.ReadAllText(dir);
30 + }
25 foreach (var commit in commits) { 31 foreach (var commit in commits) {
26 // 커밋 메시지 32 // 커밋 메시지
33 +
27 count++; 34 count++;
28 double per = ((double)count / (double)totalCount) * 100; 35 double per = ((double)count / (double)totalCount) * 100;
29 36
30 Console.Clear(); 37 Console.Clear();
31 Console.WriteLine($"{count} / {totalCount} :: {per.ToString("#0.0")}%"); 38 Console.WriteLine($"{count} / {totalCount} :: {per.ToString("#0.0")}%");
32 - 39 +
33 string message = commit.Message; 40 string message = commit.Message;
34 string cve = crawler.GetCVE(message); 41 string cve = crawler.GetCVE(message);
35 if (string.IsNullOrEmpty(cve)) { 42 if (string.IsNullOrEmpty(cve)) {
36 continue; 43 continue;
37 } 44 }
38 - 45 +
46 + string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}";
47 +
39 foreach (var parent in commit.Parents) { 48 foreach (var parent in commit.Parents) {
40 49
41 try 50 try
...@@ -49,7 +58,7 @@ namespace VulnCrawler ...@@ -49,7 +58,7 @@ namespace VulnCrawler
49 var dsp = dirPath.Split(Path.DirectorySeparatorChar); 58 var dsp = dirPath.Split(Path.DirectorySeparatorChar);
50 string repoName = dsp[dsp.Length - 1]; 59 string repoName = dsp[dsp.Length - 1];
51 // 현재 커밋에 대한 패치 엔트리 배열을 출력함 60 // 현재 커밋에 대한 패치 엔트리 배열을 출력함
52 - PrintPatchEntrys(entrys, crawler, message, cve, repoName); 61 + PrintPatchEntrys(entrys, crawler, message, cve, repoName, commitUrl);
53 // Console.ReadLine(); 62 // Console.ReadLine();
54 } 63 }
55 catch(Exception) 64 catch(Exception)
...@@ -58,7 +67,7 @@ namespace VulnCrawler ...@@ -58,7 +67,7 @@ namespace VulnCrawler
58 } 67 }
59 } 68 }
60 69
61 - private static void PrintPatchEntrys(IEnumerable<PatchEntryChanges> entrys, VulnAbstractCrawler self, string commitMsg, string cve, string repoName) { 70 + private static void PrintPatchEntrys(IEnumerable<PatchEntryChanges> entrys, VulnAbstractCrawler self, string commitMsg, string cve, string repoName, string commitUrl) {
62 foreach (var entry in entrys) { 71 foreach (var entry in entrys) {
63 // 기존 소스코드 72 // 기존 소스코드
64 var oldOid = entry.OldOid; 73 var oldOid = entry.OldOid;
...@@ -78,33 +87,6 @@ namespace VulnCrawler ...@@ -78,33 +87,6 @@ namespace VulnCrawler
78 // 출력 87 // 출력
79 if (regs.Count > 0) 88 if (regs.Count > 0)
80 { 89 {
81 - //int deleted = entry.LinesDeleted;
82 - //if (deleted == 0)
83 - //{
84 - // // continue;
85 - //}
86 - //Console.BackgroundColor = ConsoleColor.DarkBlue;
87 - //Console.WriteLine($"Old Content: \n{oldContent}");
88 - //Console.ResetColor();
89 -
90 - //Console.ForegroundColor = ConsoleColor.Blue;
91 - //Console.WriteLine($"status: {entry.Status.ToString()}");
92 - //Console.WriteLine($"added: {entry.LinesAdded.ToString()}, deleted: {entry.LinesDeleted.ToString()}");
93 - //Console.WriteLine($"old path: {entry.OldPath.ToString()}, new path: {entry.Path.ToString()}");
94 - //Console.ResetColor();
95 -
96 -
97 - //Console.Write($"CVE: ");
98 - //Console.ForegroundColor = ConsoleColor.Red;
99 - //Console.Write($"{cve}");
100 - //Console.WriteLine("");
101 - //Console.ResetColor();
102 - //Console.ForegroundColor = ConsoleColor.Yellow;
103 - //Console.WriteLine($"Commit Message: {commitMsg}");
104 - //Console.ResetColor();
105 - //Console.BackgroundColor = ConsoleColor.DarkRed;
106 - //Console.WriteLine($"Patched: \n{entry.Patch}");
107 - //Console.ResetColor();
108 /* 패치된 코드들에서 Method로 나누고 크리티컬 변수로 뽑아옴 Dictionary 구조 (키 = 함수명) */ 90 /* 패치된 코드들에서 Method로 나누고 크리티컬 변수로 뽑아옴 Dictionary 구조 (키 = 함수명) */
109 var table = self.ExtractGitCriticalMethodTable(entry.Patch); 91 var table = self.ExtractGitCriticalMethodTable(entry.Patch);
110 /* 크리티컬 메서드 테이블과 패치 전 파일에서 Process 하고 tuple로 가져옴 */ 92 /* 크리티컬 메서드 테이블과 패치 전 파일에서 Process 하고 tuple로 가져옴 */
...@@ -112,80 +94,32 @@ namespace VulnCrawler ...@@ -112,80 +94,32 @@ namespace VulnCrawler
112 { 94 {
113 /* 메서드 이름, 원본 함수 코드, 블록 리스트(크리티컬 포함) */ 95 /* 메서드 이름, 원본 함수 코드, 블록 리스트(크리티컬 포함) */
114 (var methodName, var oriFunc, var blocks) = tuple; 96 (var methodName, var oriFunc, var blocks) = tuple;
115 - //Console.BackgroundColor = ConsoleColor.DarkRed;
116 - //Console.WriteLine($"메서드 이름 : {methodName}");
117 - //Console.ResetColor();
118 - ////foreach (var block in blocks)
119 - //{
120 - // /* 크리티컬 블록이 아니면 볼 필요 없으니 넘어감 */
121 - // if (!block.HasCritical)
122 - // {
123 - // // Console.WriteLine("크리티컬 아님");
124 - // continue;
125 - // }
126 -
127 -
128 - // if (block.HasCritical)
129 - // {
130 - // Console.BackgroundColor = ConsoleColor.DarkMagenta;
131 - // }
132 - // else
133 - // {
134 - // Console.BackgroundColor = ConsoleColor.DarkGreen;
135 - // }
136 - // /* 블록 정보 출력(블록 번호, 블록 소스코드, 블록 추상화 코드, 블록 해쉬값) */
137 - // Console.WriteLine($"=====block({block.Num}, {block.HasCritical.ToString()})");
138 - // Console.WriteLine(block.Code);
139 - // Console.ResetColor();
140 - // Console.WriteLine($"AbsCode = \n{block.AbsCode}");
141 - // Console.WriteLine($"MD5 = {block.Hash}");
142 -
143 - // /* base64 인코딩(MySQL에 들어갈 수 없는 문자열이 있을 수 있으므로 인코딩) */
144 - // byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName);
145 - // byte[] codeOriBeforeBytes = Encoding.Unicode.GetBytes(oriFunc);
146 - // byte[] codeAbsBeforeBytes = Encoding.Unicode.GetBytes(block.AbsCode);
147 -
148 - // /* VulnDB에 하나의 레코드로 들어가는 하나의 취약점 객체 */
149 - // VulnRDS.Vuln vuln = new VulnRDS.Vuln()
150 - // {
151 - // Cve = cve,
152 - // BlockHash = block.Hash,
153 - // LenBlock = block.Code.Length,
154 - // FuncName = Convert.ToBase64String(funcNameBytes),
155 - // //CodeOriBefore = Convert.ToBase64String(codeOriBeforeBytes),
156 - // //CodeAbsBefore = Convert.ToBase64String(codeAbsBeforeBytes),
157 - // //NumBlock = block.Num,
158 - // };
159 - // Console.WriteLine($"Vuln FuncName:{vuln.FuncName}");
160 - /* VulnDB에 추가 */
161 - //VulnRDS.InsertVulnData(vuln);
162 - //}
163 97
164 if (string.IsNullOrWhiteSpace(oriFunc)) 98 if (string.IsNullOrWhiteSpace(oriFunc))
165 { 99 {
166 continue; 100 continue;
167 } 101 }
102 +
103 +
168 string abstractCode = self.Abstract(oriFunc, new Dictionary<string, string>(), new Dictionary<string, string>()); 104 string abstractCode = self.Abstract(oriFunc, new Dictionary<string, string>(), new Dictionary<string, string>());
169 105
170 byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName); 106 byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName);
171 byte[] absCodeBytes = Encoding.Unicode.GetBytes(abstractCode); 107 byte[] absCodeBytes = Encoding.Unicode.GetBytes(abstractCode);
172 - byte[] commitMsgBytes = Encoding.Unicode.GetBytes(commitMsg); 108 + byte[] commitUrlBytes = Encoding.Unicode.GetBytes(commitUrl);
173 byte[] funcBytes = Encoding.Unicode.GetBytes(oriFunc); 109 byte[] funcBytes = Encoding.Unicode.GetBytes(oriFunc);
110 +
111 + string absCodeBase64 = Convert.ToBase64String(absCodeBytes);
112 +
174 VulnRDS._Vuln vuln = new VulnRDS._Vuln() 113 VulnRDS._Vuln vuln = new VulnRDS._Vuln()
175 { 114 {
176 - LenFunc = oriFunc.Length, 115 + LenFunc = absCodeBase64.Length,
177 Cve = cve, 116 Cve = cve,
178 - BlockHash = VulnAbstractCrawler.MD5HashFunc(Convert.ToBase64String(absCodeBytes)), 117 + BlockHash = VulnAbstractCrawler.MD5HashFunc(absCodeBase64),
179 FuncName = Convert.ToBase64String(funcNameBytes), 118 FuncName = Convert.ToBase64String(funcNameBytes),
180 Code = Convert.ToBase64String(funcBytes), 119 Code = Convert.ToBase64String(funcBytes),
181 - Url = Convert.ToBase64String(commitMsgBytes), 120 + Url = Convert.ToBase64String(commitUrlBytes),
182 - //BlockHash = Convert.ToBase64String(absCodeBytes),
183 - //Cve = cve,
184 - //LenBlock = oriFunc.Length,
185 - //FuncName = Convert.ToBase64String(funcNameBytes),
186 }; 121 };
187 - // Console.WriteLine(vuln.BlockHash); 122 +
188 - // Console.ReadLine();
189 /* VulnDB에 추가 */ 123 /* VulnDB에 추가 */
190 VulnRDS._InsertVulnData(vuln); 124 VulnRDS._InsertVulnData(vuln);
191 125
...@@ -193,8 +127,6 @@ namespace VulnCrawler ...@@ -193,8 +127,6 @@ namespace VulnCrawler
193 } 127 }
194 else 128 else
195 { 129 {
196 - //Console.WriteLine("zzz");
197 - //Console.ReadLine();
198 continue; 130 continue;
199 } 131 }
200 132
...@@ -203,8 +135,6 @@ namespace VulnCrawler ...@@ -203,8 +135,6 @@ namespace VulnCrawler
203 } 135 }
204 catch (Exception e) 136 catch (Exception e)
205 { 137 {
206 - //Console.WriteLine(e.ToString());
207 - //Console.ReadLine();
208 continue; 138 continue;
209 } 139 }
210 140
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
2 using BloomFilter; 2 using BloomFilter;
3 using System; 3 using System;
4 using System.Collections.Generic; 4 using System.Collections.Generic;
5 +using System.Diagnostics;
5 using System.IO; 6 using System.IO;
6 using System.Linq; 7 using System.Linq;
7 using System.Text; 8 using System.Text;
...@@ -15,14 +16,17 @@ namespace VulnUserCodeAnalyzer ...@@ -15,14 +16,17 @@ namespace VulnUserCodeAnalyzer
15 { 16 {
16 static void Main(string[] args) 17 static void Main(string[] args)
17 { 18 {
19 + var crawler = new VulnC();
20 + //var bytes = Convert.FromBase64String("dgBvAGkAZAAgAGsAdgBtAF8AbQBtAHUAXwBuAGUAdwBfAGMAcgAzACgAcwB0AHIAdQBjAHQAIABrAHYAbQBfAHYAYwBwAHUAIAAqAHYAYwBwAHUAKQANAAoAewANAAoACQBtAG0AdQBfAGYAcgBlAGUAXwByAG8AbwB0AHMAKAB2AGMAcAB1ACkAOwANAAoAfQANAAoA");
21 + //var str = Encoding.Unicode.GetString(bytes);
22 +
23 + //Console.WriteLine(str);
24 + //Console.WriteLine(crawler.Abstract(str, new Dictionary<string, string>(), new Dictionary<string, string>()));
25 + //Console.ReadLine();
18 26
19 // default usage 27 // default usage
20 - int capacity = 20000000; 28 + int capacity = 50000000;
21 var filter = new Filter<string>(capacity); 29 var filter = new Filter<string>(capacity);
22 - //filter.Add("1");
23 - // filter.Add("1");
24 - //Console.WriteLine(filter.Contains("1"));
25 - //Console.WriteLine(filter.Contains("content2"));
26 30
27 /* AWS 계정 정보 파일 읽음 */ 31 /* AWS 계정 정보 파일 읽음 */
28 string txt = File.ReadAllText(@"Account.xml"); 32 string txt = File.ReadAllText(@"Account.xml");
...@@ -42,6 +46,7 @@ namespace VulnUserCodeAnalyzer ...@@ -42,6 +46,7 @@ namespace VulnUserCodeAnalyzer
42 catch (Exception e) 46 catch (Exception e)
43 { 47 {
44 Console.WriteLine($"접속 에러 :: {e.ToString()}"); 48 Console.WriteLine($"접속 에러 :: {e.ToString()}");
49 + return;
45 } 50 }
46 51
47 /* AWS 연결 여부 확인 */ 52 /* AWS 연결 여부 확인 */
...@@ -58,13 +63,16 @@ namespace VulnUserCodeAnalyzer ...@@ -58,13 +63,16 @@ namespace VulnUserCodeAnalyzer
58 63
59 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>(); 64 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
60 65
66 + Stopwatch stopwatch = new Stopwatch();
67 + stopwatch.Start();
61 DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code"); 68 DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code");
62 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories); 69 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
63 int totalFileCount = codeFiles.Count(); 70 int totalFileCount = codeFiles.Count();
64 - var crawler = new VulnC(); 71 +
65 int count = 0; 72 int count = 0;
66 foreach (var codeFile in codeFiles) 73 foreach (var codeFile in codeFiles)
67 { 74 {
75 +
68 Console.WriteLine(codeFile.FullName); 76 Console.WriteLine(codeFile.FullName);
69 using (var reader = codeFile.OpenText()) 77 using (var reader = codeFile.OpenText())
70 { 78 {
...@@ -91,14 +99,16 @@ namespace VulnUserCodeAnalyzer ...@@ -91,14 +99,16 @@ namespace VulnUserCodeAnalyzer
91 Console.Clear(); 99 Console.Clear();
92 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}"); 100 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}");
93 101
94 - //if (count > 100) 102 + if (count > 100)
95 - //{ 103 + {
96 - // break; 104 + break;
97 - //} 105 + }
98 } 106 }
99 107
100 108
101 } 109 }
110 +
111 + var findBlocks = new Queue<VulnAbstractCrawler.UserBlock>();
102 112
103 foreach (var set in hashDict) 113 foreach (var set in hashDict)
104 { 114 {
...@@ -118,10 +128,8 @@ namespace VulnUserCodeAnalyzer ...@@ -118,10 +128,8 @@ namespace VulnUserCodeAnalyzer
118 Console.WriteLine("userBlock이 비어있습니다."); 128 Console.WriteLine("userBlock이 비어있습니다.");
119 continue; 129 continue;
120 } 130 }
121 - 131 + Console.WriteLine($"CVE:{vuln.Cve}, {userBlock.FuncName}, 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}");
122 - Console.WriteLine($"{userBlock.FuncName} 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}"); 132 + findBlocks.Enqueue(userBlock);
123 -
124 -
125 } 133 }
126 } 134 }
127 135
...@@ -133,6 +141,17 @@ namespace VulnUserCodeAnalyzer ...@@ -133,6 +141,17 @@ namespace VulnUserCodeAnalyzer
133 //} 141 //}
134 } 142 }
135 143
144 + stopwatch.Stop();
145 +
146 +
147 +
148 +
149 + var hours = stopwatch.Elapsed.TotalHours;
150 + var minutes = stopwatch.Elapsed.TotalMinutes;
151 + var seconds = stopwatch.Elapsed.TotalSeconds;
152 +
153 + Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
154 +
136 155
137 // 블룸 필터 테스트 156 // 블룸 필터 테스트
138 //while(true) 157 //while(true)
......