노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
이경수
31f27046 2 parents c25155a8 ddbc5576

Merge branch 'master' of https://github.com/yhackerbv/VulnNottiProject

Inline Side-by-side
Showing 10 changed files with 712 additions and 246 deletions
...@@ -25,21 +25,14 @@ namespace DownloaderGithubClone ...@@ -25,21 +25,14 @@ namespace DownloaderGithubClone
25 Console.Write("Git Repository URL을 입력하세요 : "); 25 Console.Write("Git Repository URL을 입력하세요 : ");
26 string url = Console.ReadLine(); 26 string url = Console.ReadLine();
27 //https://github.com/django/django.git 27 //https://github.com/django/django.git
28 -
29 -
30 string pattern = @"https://github.com/.+/(?<ProjectName>.+)\.(.+)"; 28 string pattern = @"https://github.com/.+/(?<ProjectName>.+)\.(.+)";
31 -
32 var match = Regex.Match(url, pattern); 29 var match = Regex.Match(url, pattern);
33 -
34 if (!match.Success) { 30 if (!match.Success) {
35 Console.WriteLine($"패턴이 맞지 않습니다. Pattern : {pattern}"); 31 Console.WriteLine($"패턴이 맞지 않습니다. Pattern : {pattern}");
36 return; 32 return;
37 } 33 }
38 -
39 -
40 string prName = match.Groups["ProjectName"].Value; 34 string prName = match.Groups["ProjectName"].Value;
41 Console.WriteLine(prName); 35 Console.WriteLine(prName);
42 -
43 int idx = 1; 36 int idx = 1;
44 string path = Path.Combine(dir, prName); 37 string path = Path.Combine(dir, prName);
45 if (Directory.Exists(path)) { 38 if (Directory.Exists(path)) {
......
...@@ -64,7 +64,7 @@ namespace VulnCrawler ...@@ -64,7 +64,7 @@ namespace VulnCrawler
64 } 64 }
65 65
66 /* 메인 동작 함수 */ 66 /* 메인 동작 함수 */
67 - public static void Run() { 67 + public static void Run() {
68 // Repository 폴더들이 있는 주소를 지정하면 하위 폴더 목록을 가져옴(Repository 목록) 68 // Repository 폴더들이 있는 주소를 지정하면 하위 폴더 목록을 가져옴(Repository 목록)
69 Regex.CacheSize = 50; 69 Regex.CacheSize = 50;
70 70
...@@ -80,17 +80,17 @@ namespace VulnCrawler ...@@ -80,17 +80,17 @@ namespace VulnCrawler
80 // Repository 목록 만큼 반복함. 80 // Repository 목록 만큼 반복함.
81 foreach (var directory in directorys) { 81 foreach (var directory in directorys) {
82 /* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */ 82 /* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */
83 - if (directory.Contains("linux")) 83 + if (directory.Contains("~"))
84 { 84 {
85 - // continue; 85 + continue;
86 } 86 }
87 // 템플릿 패턴화 T : VulnAbstractCrawler 87 // 템플릿 패턴화 T : VulnAbstractCrawler
88 VulnWorker.Run<VulnC>(directory); 88 VulnWorker.Run<VulnC>(directory);
89 } 89 }
90 stopwatch.Stop(); 90 stopwatch.Stop();
91 - var hours = stopwatch.Elapsed.TotalHours; 91 + var hours = stopwatch.Elapsed.Hours;
92 - var minutes = stopwatch.Elapsed.TotalMinutes; 92 + var minutes = stopwatch.Elapsed.Minutes;
93 - var seconds = stopwatch.Elapsed.TotalSeconds; 93 + var seconds = stopwatch.Elapsed.Seconds;
94 94
95 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}"); 95 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
96 96
......
...@@ -31,7 +31,7 @@ namespace VulnCrawler ...@@ -31,7 +31,7 @@ namespace VulnCrawler
31 public string FuncName { get; set; } 31 public string FuncName { get; set; }
32 public string Hash { get; set; } 32 public string Hash { get; set; }
33 public string Path { get; set; } 33 public string Path { get; set; }
34 - 34 + public string Url { get; set; }
35 public override bool Equals(object obj) 35 public override bool Equals(object obj)
36 { 36 {
37 var block = obj as UserBlock; 37 var block = obj as UserBlock;
...@@ -131,14 +131,15 @@ namespace VulnCrawler ...@@ -131,14 +131,15 @@ namespace VulnCrawler
131 /// <summary> 131 /// <summary>
132 /// 커밋에서 검색할 정규식 문자열 132 /// 커밋에서 검색할 정규식 문자열
133 /// </summary> 133 /// </summary>
134 - public string SearchCommitPattern => @"CVE[ -]\d{4}[ -]\d{4}"; 134 + public string SearchCommitPattern => @"CVE[ -](201[5-8])[ -](\d{4,})";
135 /// <summary> 135 /// <summary>
136 /// 패치 코드에서 함수 찾을 정규식 패턴 문자열 136 /// 패치 코드에서 함수 찾을 정규식 패턴 문자열
137 /// </summary> 137 /// </summary>
138 protected abstract string RegexFuncPattern { get; } 138 protected abstract string RegexFuncPattern { get; }
139 + protected abstract string UserRegexFuncPattern { get; }
139 protected abstract string Extension { get; } 140 protected abstract string Extension { get; }
140 public virtual IEnumerable<PatchEntryChanges> GetPatchEntryChanges(Patch patch) { 141 public virtual IEnumerable<PatchEntryChanges> GetPatchEntryChanges(Patch patch) {
141 - return patch.Where(e => e.Path.EndsWith(Extension)).ToList(); 142 + return patch.Where(e => e.Path.EndsWith(Extension) && e.Status == ChangeKind.Modified).ToList();
142 } 143 }
143 /// <summary> 144 /// <summary>
144 /// 정규식을 이용하여 @@ -\d,\d +\d,\d @@ MethodName(): 이런 패턴을 찾고 145 /// 정규식을 이용하여 @@ -\d,\d +\d,\d @@ MethodName(): 이런 패턴을 찾고
...@@ -158,108 +159,11 @@ namespace VulnCrawler ...@@ -158,108 +159,11 @@ namespace VulnCrawler
158 public abstract IDictionary<int, IEnumerable<UserBlock>> CrawlUserCode(StreamReader reader); 159 public abstract IDictionary<int, IEnumerable<UserBlock>> CrawlUserCode(StreamReader reader);
159 160
160 protected abstract IList<Block> GetCriticalBlocks(string srcCode, IEnumerable<string> criticalList); 161 protected abstract IList<Block> GetCriticalBlocks(string srcCode, IEnumerable<string> criticalList);
161 - /// <summary>
162 - /// 성능 개선을 위한
163 - /// 코드 라인 위치 기반 취약 원본 함수 추출 테스트용 함수 곧 삭제 예정
164 - /// </summary>
165 - public string GetOriginalFuncTest(Stream oldStream, string methodName, int start)
166 - {
167 - StringBuilder oldBuilder = new StringBuilder();
168 -
169 - using (var reader = new StreamReader(oldStream))
170 - {
171 - bool found = false;
172 - bool found2 = false;
173 - bool commentLine = false;
174 - int bracketCount = -1;
175 - string stringPattern = @"[""].*[""]";
176 - string commentPattern = @"\/\*.+\*\/";
177 - string commentPattern2 = @"\/\*";
178 - string commentPattern3 = @"\*\/";
179 - int readCount = 0;
180 - Queue<string> tempQ = new Queue<string>();
181 - while (!reader.EndOfStream)
182 - {
183 - string line = reader.ReadLine();
184 - if (readCount++ < start)
185 - {
186 - tempQ.Enqueue(line);
187 - continue;
188 - }
189 - Stack<string> tempStack = new Stack<string>();
190 - while (tempQ.Count > 0)
191 - {
192 - string s = tempQ.Dequeue();
193 - tempStack.Push(s);
194 - string method = Regex.Escape(methodName);
195 - if (Regex.Match(s, $"{method}").Success)
196 - {
197 - break;
198 - }
199 - }
200 - while (tempStack.Count > 0)
201 - {
202 - string s = tempStack.Pop();
203 - string trim = s.Trim();
204 - if (commentLine)
205 - {
206 - if (Regex.IsMatch(trim, commentPattern3))
207 - {
208 - commentLine = false;
209 - trim = Regex.Split(trim, commentPattern3)[1];
210 - }
211 - continue;
212 - }
213 - string removeString = Regex.Replace(trim, stringPattern, "");
214 - // /* ~ 패턴
215 - if (Regex.IsMatch(trim, commentPattern2))
216 - {
217 - // /* ~ */ 패턴이 아닌 경우
218 - if (!Regex.IsMatch(trim, commentPattern))
219 - {
220 - commentLine = true;
221 - }
222 - trim = Regex.Split(trim, "/*")[0];
223 - }
224 - if (string.IsNullOrWhiteSpace(trim))
225 - {
226 - continue;
227 - }
228 - int openBracketCount = removeString.Count(c => c == '{');
229 - int closeBracketCount = removeString.Count(c => c == '}');
230 - int subtract = openBracketCount - closeBracketCount;
231 - bracketCount += subtract;
232 - // 메서드 시작 괄호 찾은 경우
233 - if (found2)
234 - {
235 - // 괄호가 모두 닫혔으니 종료
236 - if (bracketCount < 0)
237 - {
238 - // Console.WriteLine("괄호끝");
239 - break;
240 - }
241 - // oldBuilder.AppendLine(line);
242 - }
243 - else
244 - {
245 - if (openBracketCount > 0)
246 - {
247 - found2 = true;
248 - }
249 -
250 - }
251 - oldBuilder.AppendLine(s);
252 - }
253 - }
254 - }
255 - Console.WriteLine("찾음");
256 - Console.WriteLine(oldBuilder.ToString());
257 - Console.ReadLine();
258 162
259 - return oldBuilder.ToString();
260 - }
261 public abstract IDictionary<string, IEnumerable<string>> ExtractGitCriticalMethodTable(string srcCode); 163 public abstract IDictionary<string, IEnumerable<string>> ExtractGitCriticalMethodTable(string srcCode);
262 164
165 + public abstract IDictionary<string, string> CrawlCode(StreamReader reader);
166 +
263 public abstract string Abstract(string blockCode, IDictionary<string, string> dict, IDictionary<string, string> methodDict); 167 public abstract string Abstract(string blockCode, IDictionary<string, string> dict, IDictionary<string, string> methodDict);
264 /// <summary> 168 /// <summary>
265 /// 패치 전 코드 파일과 크리티컬 메서드 테이블로 부터 크리티컬 블록 추출 169 /// 패치 전 코드 파일과 크리티컬 메서드 테이블로 부터 크리티컬 블록 추출
...@@ -268,54 +172,42 @@ namespace VulnCrawler ...@@ -268,54 +172,42 @@ namespace VulnCrawler
268 /// <param name="table">크리티컬 메서드 테이블(Key: 메서드 이름, Value: 변수 리스트)</param> 172 /// <param name="table">크리티컬 메서드 테이블(Key: 메서드 이름, Value: 변수 리스트)</param>
269 /// <returns></returns> 173 /// <returns></returns>
270 public virtual IEnumerable<(string methodName, string oriFunc, IList<Block> blocks)> Process(Blob oldBlob, IDictionary<string, IEnumerable<string>> table) { 174 public virtual IEnumerable<(string methodName, string oriFunc, IList<Block> blocks)> Process(Blob oldBlob, IDictionary<string, IEnumerable<string>> table) {
271 - foreach (var item in table) 175 +
176 + // 패치 전 원본 파일 스트림
177 + Stream oldStream = oldBlob.GetContentStream();
178 + using (var reader = new StreamReader(oldStream))
272 { 179 {
273 - var methodTable = new Dictionary<string, string>(); 180 + var dict = CrawlCode(reader);
274 - var varTable = new Dictionary<string, string>(); 181 +
275 - // 메서드 이름 182 + foreach (var item in table)
276 - string methodName = item.Key;
277 - // 패치 전 원본 파일 스트림
278 - Stream oldStream = oldBlob.GetContentStream();
279 - // 패치 전 원본 함수 구하고
280 - string func = GetOriginalFunc(oldStream, methodName);
281 -
282 - string bs = string.Empty;
283 - string md5 = string.Empty;
284 - if (item.Value.Count() != 0)
285 { 183 {
286 - //Console.WriteLine("크리티컬 변수 목록"); 184 + var methodTable = new Dictionary<string, string>();
287 - //Console.ForegroundColor = ConsoleColor.Cyan; 185 + var varTable = new Dictionary<string, string>();
288 - //foreach (var c in item.Value) 186 + // 메서드 이름
289 - //{ 187 + string methodName = item.Key;
290 - // Console.WriteLine(c); 188 +
291 - //} 189 + // 패치 전 원본 함수 구하고
292 - //Console.ResetColor(); 190 + string func = string.Empty;
293 - //Console.WriteLine("-------------------"); 191 +
192 +
193 + foreach (var pair in dict)
194 + {
195 + if (pair.Key.Contains(methodName))
196 + {
197 + func = pair.Value;
198 + break;
199 + }
200 + }
201 +
202 +
203 +
204 +
294 // 크리티컬 블록 추출 205 // 크리티컬 블록 추출
295 var blocks = new List<Block>(); 206 var blocks = new List<Block>();
296 - //var blocks = GetCriticalBlocks(func, item.Value).ToList();
297 - //if (blocks == null)
298 - //{
299 - // continue;
300 - //}
301 - //foreach (var block in blocks)
302 - //{
303 -
304 - // block.CriticalList = item.Value;
305 - // /* 추상화 및 정규화 */
306 - // block.AbsCode = Abstract(block.Code, varTable, methodTable);
307 - // block.Hash = MD5HashFunc(block.AbsCode);
308 -
309 - //}
310 - /* 추상화 변환 테이블 출력 */
311 - //foreach (var var in varTable)
312 - //{
313 - // Console.WriteLine($"{var.Key}, {var.Value}");
314 - //}
315 -
316 yield return (methodName, func, blocks); 207 yield return (methodName, func, blocks);
208 +
209 +
317 } 210 }
318 -
319 } 211 }
320 } 212 }
321 /// <summary> 213 /// <summary>
...@@ -349,7 +241,7 @@ namespace VulnCrawler ...@@ -349,7 +241,7 @@ namespace VulnCrawler
349 var match = Regex.Match(msg, SearchCommitPattern, RegexOptions.IgnoreCase); 241 var match = Regex.Match(msg, SearchCommitPattern, RegexOptions.IgnoreCase);
350 242
351 if (match.Success) { 243 if (match.Success) {
352 - return match.Value; 244 + return $"CVE-{match.Groups[1].Value}-{match.Groups[2].Value}";
353 } 245 }
354 return string.Empty; 246 return string.Empty;
355 } 247 }
......
...@@ -12,7 +12,8 @@ namespace VulnCrawler ...@@ -12,7 +12,8 @@ namespace VulnCrawler
12 { 12 {
13 // protected override string RegexFuncPattern => $@"@@ \-(?<{OldStart}>\d+),(?<{OldLines}>\d+) \+(?<{NewStart}>\d+),(?<{NewLines}>\d+) @@ (?<{MethodName}>(static)?( const )? [\w]+ [\w]+\([\w \*\,\t\n]*[\)\,])"; 13 // protected override string RegexFuncPattern => $@"@@ \-(?<{OldStart}>\d+),(?<{OldLines}>\d+) \+(?<{NewStart}>\d+),(?<{NewLines}>\d+) @@ (?<{MethodName}>(static)?( const )? [\w]+ [\w]+\([\w \*\,\t\n]*[\)\,])";
14 /* 함수 패턴 정규식 */ 14 /* 함수 패턴 정규식 */
15 - protected override string RegexFuncPattern => $@"^[\w \*]*(?<{MethodName}>[\w\*]+ [\w\*]+\(([\w \*\,\t\n])*[\)\,])"; 15 + protected override string UserRegexFuncPattern => $@"^[\w \*]*(?<{MethodName}>[\w\*]+ [\w\*]+\(([\w \*\,\t\n])*[\)\,])";
16 + protected override string RegexFuncPattern => $@"(?<{MethodName}>(unsigned|static)?( const )? [\w]+ [\w]+\(([\w \*\,\t\n])*[\)\,])";
16 /* 검색 파일 타입 */ 17 /* 검색 파일 타입 */
17 protected override string Extension => ".c"; 18 protected override string Extension => ".c";
18 /* 예약어 파일명 */ 19 /* 예약어 파일명 */
...@@ -26,8 +27,7 @@ namespace VulnCrawler ...@@ -26,8 +27,7 @@ namespace VulnCrawler
26 /// <param name="patchCode">패치 코드</param> 27 /// <param name="patchCode">패치 코드</param>
27 /// <returns></returns> 28 /// <returns></returns>
28 public override MatchCollection GetMatches(string patchCode) { 29 public override MatchCollection GetMatches(string patchCode) {
29 - var funcPattern = $@"(?<{MethodName}>(unsigned|static)?( const )? [\w]+ [\w]+\(([\w \*\,\t\n])*[\)\,])"; 30 + var regs = Regex.Matches(patchCode, RegexFuncPattern);
30 - var regs = Regex.Matches(patchCode, funcPattern);
31 return regs; 31 return regs;
32 } 32 }
33 /// <summary> 33 /// <summary>
...@@ -643,11 +643,9 @@ namespace VulnCrawler ...@@ -643,11 +643,9 @@ namespace VulnCrawler
643 return temp; 643 return temp;
644 } 644 }
645 645
646 - public override IDictionary<int, IEnumerable<UserBlock>> CrawlUserCode(StreamReader reader) 646 + public override IDictionary<string, string> CrawlCode(StreamReader reader)
647 { 647 {
648 - 648 + var dict = new Dictionary<string, string>();
649 -
650 - var dict = new Dictionary<int, IEnumerable<UserBlock>>();
651 StringBuilder oldBuilder = new StringBuilder(); 649 StringBuilder oldBuilder = new StringBuilder();
652 650
653 bool found = false; 651 bool found = false;
...@@ -667,9 +665,10 @@ namespace VulnCrawler ...@@ -667,9 +665,10 @@ namespace VulnCrawler
667 665
668 bool com = false; 666 bool com = false;
669 667
668 +
670 while (!reader.EndOfStream) 669 while (!reader.EndOfStream)
671 { 670 {
672 - 671 +
673 string line = reader.ReadLine(); 672 string line = reader.ReadLine();
674 string trim = line.Trim(); 673 string trim = line.Trim();
675 if (commentLine) 674 if (commentLine)
...@@ -711,17 +710,243 @@ namespace VulnCrawler ...@@ -711,17 +710,243 @@ namespace VulnCrawler
711 if (found3) 710 if (found3)
712 { 711 {
713 string obStr = oldBuilder.ToString(); 712 string obStr = oldBuilder.ToString();
713 +
714 + string funcName = new string(obStr.TakeWhile(c => c != '{').ToArray());
715 +
716 + if (!dict.ContainsKey(funcName))
717 + {
718 + dict[funcName] = string.Empty;
719 + }
720 +
721 +
722 + dict[funcName] = obStr;
723 + oldBuilder.Clear();
724 + found = false;
725 + found2 = false;
726 + found3 = false;
727 + bracketCount = -1;
728 + commentLine = false;
729 + }
730 + if (found)
731 + {
732 + // 범위 주석 진행되고 있으면 넘어감
733 + if (trim.StartsWith("#"))
734 + {
735 + continue;
736 + }
737 + if (commentLine)
738 + {
739 + // 혹시 범위 주석이 끝났는지 체크
740 + if (regex1.IsMatch(trim))
741 + {
742 + commentLine = false;
743 + trim = regex1.Split(trim)[1];
744 + }
745 + else
746 + {
747 + continue;
748 + }
749 + }
750 + // "" 문자열 제거
751 + string removeString = regex2.Replace(trim, "");
752 + // /* ~ 패턴
753 + if (regex3.IsMatch(trim))
754 + {
755 + // /* ~ */ 패턴이 아닌 경우
756 + if (!regex4.IsMatch(trim))
757 + {
758 + commentLine = true;
759 + }
760 + trim = Regex.Split(trim, "/*")[0];
761 + }
762 + // 비어있는 경우 넘어감
763 + if (string.IsNullOrWhiteSpace(trim))
764 + {
765 + continue;
766 + }
767 + int openBracketCount = removeString.Count(c => c == '{');
768 + int closeBracketCount = removeString.Count(c => c == '}');
769 + int subtract = openBracketCount - closeBracketCount;
770 + bracketCount += subtract;
771 + // 메서드 시작 괄호 찾은 경우
772 + if (found2)
773 + {
774 + oldBuilder.AppendLine(line);
775 + // 괄호가 모두 닫혔으니 종료
776 + if (bracketCount < 0)
777 + {
778 + found3 = true;
779 + continue;
780 + }
781 + }
782 + else // 메서드는 찾았으나 아직 시작 괄호를 못찾은 경우
783 + {
784 + oldBuilder.AppendLine(line);
785 + if (openBracketCount > 0)
786 + {
787 +
788 + found2 = true;
789 + }
790 + else
791 + {
792 + //아직 { 괄호를 못찾았는데 );를 만났다면 메서드 선언 부분이니 넘어감
793 + if (trim.EndsWith(");"))
794 + {
795 + found = false;
796 + oldBuilder.Clear();
797 + continue;
798 + }
799 + }
800 + }
801 + }
802 + // 아직 메서드를 못찾은 경우
803 + else
804 + {
805 + //아직 { 괄호를 못찾았는데 );를 만났다면 메서드 선언 부분이니 넘어감
806 + if (line.Trim().EndsWith(");"))
807 + {
808 + found = false;
809 + oldBuilder.Clear();
810 + continue;
811 + }
812 +
813 + // 메서드 찾았는지 확인
814 + if (Regex.IsMatch(line, UserRegexFuncPattern))
815 + {
816 +
817 + // 주석으로 시작했다면 넘어감
818 + if (trim.StartsWith("//"))
819 + {
820 + continue;
821 + }
822 +
823 + if (trim.StartsWith("/*"))
824 + {
825 + com = true;
826 + continue;
827 + }
828 +
829 + // 만약 찾은 메서드 라인에서 중괄호 {가 시작된 경우
830 + if (trim.Contains("{"))
831 + {
832 + // 동시에 } 닫히기까지 한 경우 드물겠지만..
833 + if (trim.EndsWith("}"))
834 + {
835 + oldBuilder.AppendLine(line);
836 + found3 = true;
837 + continue;
838 + }
839 + found2 = true;
840 + }
841 + // 메서드 찾음
842 + found = true;
843 + oldBuilder.AppendLine(line);
844 + }
845 + }
846 +
847 + }
848 +
849 + if (found3)
850 + {
851 + string obStr = oldBuilder.ToString();
852 +
853 + string funcName = new string(obStr.TakeWhile(c => c != '{').ToArray());
854 +
855 + if (!dict.ContainsKey(funcName))
856 + {
857 + dict[funcName] = string.Empty;
858 + }
859 +
860 + dict[funcName] = obStr;
861 + oldBuilder.Clear();
862 + found = false;
863 + found2 = false;
864 + found3 = false;
865 + bracketCount = -1;
866 + commentLine = false;
867 +
868 +
869 + }
870 +
871 +
872 + return dict;
873 +
874 +
875 + }
876 +
877 + public override IDictionary<int, IEnumerable<UserBlock>> CrawlUserCode(StreamReader reader)
878 + {
879 + var dict = new Dictionary<int, IEnumerable<UserBlock>>();
880 + StringBuilder oldBuilder = new StringBuilder();
881 +
882 + bool found = false;
883 + bool found2 = false;
884 + bool commentLine = false;
885 + int bracketCount = -1;
886 + string stringPattern = @"[""].*[""]";
887 + string commentPattern = @"\/\*.+\*\/";
888 + string commentPattern2 = @"\/\*";
889 + string commentPattern3 = @"\*\/";
890 + var regex1 = new Regex(commentPattern3, RegexOptions.Compiled);
891 + var regex2 = new Regex(stringPattern, RegexOptions.Compiled);
892 + var regex3 = new Regex(commentPattern2, RegexOptions.Compiled);
893 + var regex4 = new Regex(commentPattern, RegexOptions.Compiled);
894 + bool found3 = false;
895 + bool com = false;
896 + while (!reader.EndOfStream)
897 + {
898 + string line = reader.ReadLine();
899 + string trim = line.Trim();
900 + if (commentLine)
901 + {
902 + // 혹시 범위 주석이 끝났는지 체크
903 + if (regex1.IsMatch(trim))
904 + {
905 + commentLine = false;
906 + trim = regex1.Split(trim)[1];
907 + }
908 + else
909 + {
910 + continue;
911 + }
912 + }
913 + // /* ~ 패턴
914 + if (regex3.IsMatch(trim))
915 + {
916 + // /* ~ */ 패턴이 아닌 경우
917 + if (!regex4.IsMatch(trim))
918 + {
919 + commentLine = true;
920 + }
921 + trim = Regex.Split(trim, "/*")[0];
922 + }
923 + if (com)
924 + {
925 + if (trim.StartsWith("*"))
926 + {
927 + continue;
928 + }
929 + else
930 + {
931 + com = false;
932 + }
933 + }
934 + // 메서드를 찾은 경우
935 + if (found3)
936 + {
937 + string obStr = oldBuilder.ToString();
938 + //Console.WriteLine(obStr);
714 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>()); 939 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>());
715 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr); 940 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
716 string absObStrBase64 = Convert.ToBase64String(obStrBytes); 941 string absObStrBase64 = Convert.ToBase64String(obStrBytes);
717 - 942 + // Console.WriteLine(obStr);
943 + //Console.WriteLine("HASH: " + MD5HashFunc(obStr));
944 + //Console.WriteLine(absObStrBase64);
718 if (!dict.ContainsKey(absObStrBase64.Length)) 945 if (!dict.ContainsKey(absObStrBase64.Length))
719 { 946 {
720 dict[absObStrBase64.Length] = new HashSet<UserBlock>(); 947 dict[absObStrBase64.Length] = new HashSet<UserBlock>();
721 } 948 }
722 -
723 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray()); 949 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray());
724 -
725 (dict[absObStrBase64.Length] as HashSet<UserBlock>).Add(new UserBlock 950 (dict[absObStrBase64.Length] as HashSet<UserBlock>).Add(new UserBlock
726 { 951 {
727 Hash = MD5HashFunc(absObStrBase64), 952 Hash = MD5HashFunc(absObStrBase64),
...@@ -819,7 +1044,7 @@ namespace VulnCrawler ...@@ -819,7 +1044,7 @@ namespace VulnCrawler
819 } 1044 }
820 1045
821 // 메서드 찾았는지 확인 1046 // 메서드 찾았는지 확인
822 - if (Regex.IsMatch(line, RegexFuncPattern)) 1047 + if (Regex.IsMatch(line, UserRegexFuncPattern))
823 { 1048 {
824 1049
825 // 주석으로 시작했다면 넘어감 1050 // 주석으로 시작했다면 넘어감
...@@ -857,10 +1082,11 @@ namespace VulnCrawler ...@@ -857,10 +1082,11 @@ namespace VulnCrawler
857 if (found3) 1082 if (found3)
858 { 1083 {
859 string obStr = oldBuilder.ToString(); 1084 string obStr = oldBuilder.ToString();
1085 + // Console.WriteLine(obStr);
860 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>()); 1086 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>());
861 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr); 1087 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
862 string absObStrBase64 = Convert.ToBase64String(obStrBytes); 1088 string absObStrBase64 = Convert.ToBase64String(obStrBytes);
863 - 1089 + // Console.WriteLine(obStr);
864 if (!dict.ContainsKey(absObStrBase64.Length)) 1090 if (!dict.ContainsKey(absObStrBase64.Length))
865 { 1091 {
866 dict[absObStrBase64.Length] = new HashSet<UserBlock>(); 1092 dict[absObStrBase64.Length] = new HashSet<UserBlock>();
......
...@@ -16,6 +16,9 @@ namespace VulnCrawler ...@@ -16,6 +16,9 @@ namespace VulnCrawler
16 protected override string Extension => ".py"; 16 protected override string Extension => ".py";
17 protected override string RegexFuncPattern => $@"@@ \-(?<{OldStart}>\d+),(?<{OldLines}>\d+) \+(?<{NewStart}>\d+),(?<{NewLines}>\d+) @@ def (?<{MethodName}>\w+)"; 17 protected override string RegexFuncPattern => $@"@@ \-(?<{OldStart}>\d+),(?<{OldLines}>\d+) \+(?<{NewStart}>\d+),(?<{NewLines}>\d+) @@ def (?<{MethodName}>\w+)";
18 protected override string ReservedFileName => "PyReserved.txt"; 18 protected override string ReservedFileName => "PyReserved.txt";
19 +
20 + protected override string UserRegexFuncPattern => throw new NotImplementedException();
21 +
19 public override MatchCollection GetMatches(string patchCode) { 22 public override MatchCollection GetMatches(string patchCode) {
20 //var regs = Regex.Matches(patchCode, RegexFuncPattern); 23 //var regs = Regex.Matches(patchCode, RegexFuncPattern);
21 var regs = MethodExtractor.Matches(patchCode); 24 var regs = MethodExtractor.Matches(patchCode);
...@@ -85,5 +88,10 @@ namespace VulnCrawler ...@@ -85,5 +88,10 @@ namespace VulnCrawler
85 { 88 {
86 throw new NotImplementedException(); 89 throw new NotImplementedException();
87 } 90 }
91 +
92 + public override IDictionary<string, string> CrawlCode(StreamReader reader)
93 + {
94 + throw new NotImplementedException();
95 + }
88 } 96 }
89 } 97 }
......
...@@ -53,6 +53,23 @@ namespace VulnCrawler ...@@ -53,6 +53,23 @@ namespace VulnCrawler
53 return 802558182 + EqualityComparer<string>.Default.GetHashCode(BlockHash); 53 return 802558182 + EqualityComparer<string>.Default.GetHashCode(BlockHash);
54 } 54 }
55 } 55 }
56 + public class Vuln_detail
57 + {
58 + public int Index { get; set; } = -1; /* index key */
59 + public string Type { get; set; } = "NULL"; /* type */
60 + public string Year { get; set; } = "NULL"; /* year */
61 + public string Level { get; set; } = "NULL"; /* level */
62 + public string UserName { get; set; } = "NULL"; /* user name */
63 + public string Publish_date { get; set; } = "NULL"; /* Publish_date */
64 + public string Update_date { get; set; } = "NULL"; /* Update_date */
65 + public string CveDetail { get; set; } = "NULL"; /* cveDetail */
66 + public string CveName { get; set; } = "NULL"; /* cve name */
67 + public string FileName { get; set; } = "NULL"; /* FileName */
68 + public string FuncName { get; set; } = "NULL"; /* funcName */
69 + public string Url { get; set; } = "NULL"; /* Url */
70 + public string Product { get; set; }
71 +
72 + }
56 //connect 73 //connect
57 public static void Connect(AWS.Account account, string dbName) 74 public static void Connect(AWS.Account account, string dbName)
58 { 75 {
...@@ -129,9 +146,7 @@ namespace VulnCrawler ...@@ -129,9 +146,7 @@ namespace VulnCrawler
129 { 146 {
130 last_vulnId = 1; 147 last_vulnId = 1;
131 } 148 }
132 -
133 Retry: 149 Retry:
134 -
135 //DB insert 150 //DB insert
136 try 151 try
137 { 152 {
...@@ -216,6 +231,50 @@ namespace VulnCrawler ...@@ -216,6 +231,50 @@ namespace VulnCrawler
216 Console.ReadLine(); 231 Console.ReadLine();
217 } 232 }
218 } 233 }
234 + public static void InsertVulnDetail(Vuln_detail vuln)
235 + {
236 + String sql = string.Empty;
237 + MySqlCommand cmd = null;
238 + Retry:
239 + //DB insert
240 + try
241 + {
242 + cmd = new MySqlCommand
243 + {
244 + Connection = Conn,
245 + //db에 추가
246 + CommandText = "INSERT INTO vulnDetail(type, year, level, userName, cveName, publish_date,update_date, cveDetail,fileName, funcName, url, product) VALUES(@type, @year, @level, @userName, @cveName, @publish_date,@update_date, @cveDetail,@fileName, @funcName,@url,@product)"
247 + };
248 + cmd.Parameters.AddWithValue("@type", $"{vuln.Type}");
249 + cmd.Parameters.AddWithValue("@year", $"{vuln.Year}");
250 + cmd.Parameters.AddWithValue("@level", $"{vuln.Level}");
251 + cmd.Parameters.AddWithValue("@userName", $"{vuln.UserName}");
252 + cmd.Parameters.AddWithValue("@cveName", $"{vuln.CveName}");
253 + cmd.Parameters.AddWithValue("@publish_date", $"{vuln.Publish_date}");
254 + cmd.Parameters.AddWithValue("@update_date", $"{vuln.Update_date}");
255 + cmd.Parameters.AddWithValue("@cveDetail", $"{vuln.CveDetail}");
256 + cmd.Parameters.AddWithValue("@fileName", $"{vuln.FileName}");
257 + cmd.Parameters.AddWithValue("@funcName", $"{vuln.FuncName}");
258 + cmd.Parameters.AddWithValue("@url", $"{vuln.Url}");
259 + cmd.Parameters.AddWithValue("@product", $"{vuln.Product}");
260 + cmd.ExecuteNonQuery();
261 + //콘솔출력용
262 + sql = "INSERT INTO vulnDetail(type, year, level, userName, cveName, publish_date,update_date, cveDetail,fileName, funcName, url) " +
263 + $"VALUES({vuln.Type}, {vuln.Year}, {vuln.Level}, {vuln.UserName}, {vuln.CveName},{vuln.Publish_date}, {vuln.Update_date}, {vuln.CveDetail}, {vuln.FileName}, {vuln.FuncName}, {vuln.Url})";
264 + // Console.WriteLine(sql);
265 + }
266 + catch (Exception e)
267 + {
268 + // Console.WriteLine(e.ToString());
269 + string es = e.ToString();
270 + if (es.Contains("Connection must be valid and open"))
271 + {
272 + Connect(Account, DbName);
273 + goto Retry;
274 + }
275 + }
276 + }
277 +
219 public static void UpdateVulnData(int _vulnId, _Vuln vuln) { 278 public static void UpdateVulnData(int _vulnId, _Vuln vuln) {
220 String sql = string.Empty; 279 String sql = string.Empty;
221 MySqlCommand cmd = null; 280 MySqlCommand cmd = null;
...@@ -401,33 +460,118 @@ namespace VulnCrawler ...@@ -401,33 +460,118 @@ namespace VulnCrawler
401 Console.ReadLine(); 460 Console.ReadLine();
402 } 461 }
403 } 462 }
404 - public static List<_Vuln> SelectVulnbyLen(int _lenFunc) 463 + public static IEnumerable<_Vuln> SelectVulnbyLen(int _lenFunc)
405 { 464 {
406 - var list = new List<_Vuln>();
407 String sql = string.Empty; 465 String sql = string.Empty;
408 MySqlCommand cmd = new MySqlCommand(); 466 MySqlCommand cmd = new MySqlCommand();
409 cmd.Connection = Conn; 467 cmd.Connection = Conn;
410 cmd.CommandText = "SELECT * FROM vuln_Info where lenFunc=" + _lenFunc; 468 cmd.CommandText = "SELECT * FROM vuln_Info where lenFunc=" + _lenFunc;
411 469
412 System.Data.DataSet ds = new System.Data.DataSet(); 470 System.Data.DataSet ds = new System.Data.DataSet();
413 - MySqlDataAdapter da = new MySqlDataAdapter("SELECT * FROM vuln_Info where lenFunc=" + _lenFunc, Conn); 471 + MySqlDataAdapter da = new MySqlDataAdapter(cmd.CommandText, Conn);
414 da.Fill(ds); 472 da.Fill(ds);
415 473
416 //vuln에 입력 474 //vuln에 입력
417 foreach (System.Data.DataRow row in ds.Tables[0].Rows) 475 foreach (System.Data.DataRow row in ds.Tables[0].Rows)
418 { 476 {
419 - _Vuln vuln = new _Vuln(); 477 + _Vuln vuln = new _Vuln
420 - vuln.VulnId = Convert.ToInt32(row["vulnId"]); 478 + {
421 - vuln.Cve = Convert.ToString(row["cve"]); 479 + VulnId = Convert.ToInt32(row["vulnId"]),
422 - vuln.FuncName = Convert.ToString(row["funcName"]); 480 + Cve = Convert.ToString(row["cve"]),
423 - vuln.LenFunc = Convert.ToInt32(row["lenFunc"]); 481 + FuncName = Convert.ToString(row["funcName"]),
424 - vuln.Code = Convert.ToString(row["code"]); 482 + LenFunc = Convert.ToInt32(row["lenFunc"]),
425 - vuln.BlockHash = Convert.ToString(row["blockHash"]); 483 + Code = Convert.ToString(row["code"]),
426 - vuln.Url = Convert.ToString(row["url"]); 484 + BlockHash = Convert.ToString(row["blockHash"]),
427 - list.Add(vuln); 485 + Url = Convert.ToString(row["url"])
486 + };
487 + yield return vuln;
488 + }
489 + }
490 + public static IEnumerable<_Vuln> SelectVulnbyCve(string _cve)
491 + {
492 + String sql = string.Empty;
493 + MySqlCommand cmd = new MySqlCommand();
494 + cmd.Connection = Conn;
495 + cmd.CommandText = $"SELECT * FROM vuln_Info where cve='" + _cve + $"'";
496 +
497 + System.Data.DataSet ds = new System.Data.DataSet();
498 + MySqlDataAdapter da = new MySqlDataAdapter(cmd.CommandText, Conn);
499 + da.Fill(ds);
500 + //vuln에 입력
501 + foreach (System.Data.DataRow row in ds.Tables[0].Rows)
502 + {
503 + _Vuln vuln = new _Vuln
504 + {
505 + VulnId = Convert.ToInt32(row["vulnId"]),
506 + Cve = Convert.ToString(row["cve"]),
507 + FuncName = Convert.ToString(row["funcName"]),
508 + LenFunc = Convert.ToInt32(row["lenFunc"]),
509 + Code = Convert.ToString(row["code"]),
510 + BlockHash = Convert.ToString(row["blockHash"]),
511 + Url = Convert.ToString(row["url"])
512 + };
513 + yield return vuln;
514 + }
515 + }
516 + public static IEnumerable<string> SelectRepositbyName(string _username)
517 + {
518 + String sql = string.Empty;
519 + MySqlCommand cmd = new MySqlCommand();
520 + cmd.Connection = Conn;
521 + cmd.CommandText = "SELECT repository FROM vuln.auth_user WHERE username = '" + _username + "'";
522 + string a = null;
523 +
524 + //sql console write 확인용
525 + Console.Write(cmd.CommandText);
526 +
527 + System.Data.DataSet ds = new System.Data.DataSet();
528 + MySqlDataAdapter da = new MySqlDataAdapter(cmd.CommandText, Conn);
529 + da.Fill(ds);
530 + //string을 넣음
531 + foreach (System.Data.DataRow row in ds.Tables[0].Rows)
532 + {
533 + a = Convert.ToString(row["repository"]);
534 + yield return a;
535 + }
536 + }
537 + public static IEnumerable<(string userName, string repository)> SelectAllReposit()
538 + {
539 + String sql = string.Empty;
540 + MySqlCommand cmd = new MySqlCommand
541 + {
542 + Connection = Conn,
543 + CommandText = "SELECT username, repository FROM vuln.auth_user "
544 + };
545 + System.Data.DataSet ds = new System.Data.DataSet();
546 + MySqlDataAdapter da = new MySqlDataAdapter(cmd.CommandText, Conn);
547 + da.Fill(ds);
548 + //vuln에 입력
549 + foreach (System.Data.DataRow row in ds.Tables[0].Rows)
550 + {
551 + string repo = Convert.ToString(row["repository"]);
552 + string user = Convert.ToString(row["username"]);
553 + yield return (user, repo);
554 + }
555 + }
556 + public static IEnumerable<string> SelectReposit_detail()
557 + {
558 + String sql = string.Empty;
559 + MySqlCommand cmd = new MySqlCommand();
560 + cmd.Connection = Conn;
561 + cmd.CommandText = "SELECT url FROM vulnDetail ";
562 + string a = null;
563 +
564 + System.Data.DataSet ds = new System.Data.DataSet();
565 + MySqlDataAdapter da = new MySqlDataAdapter(cmd.CommandText, Conn);
566 + da.Fill(ds);
567 + //vuln에 입력
568 + foreach (System.Data.DataRow row in ds.Tables[0].Rows)
569 + {
570 + a = Convert.ToString(row["url"]);
571 + Console.WriteLine(a);
572 +
573 + yield return a;
428 } 574 }
429 - //해당 list 반환
430 - return list;
431 } 575 }
432 576
433 } 577 }
......
...@@ -30,7 +30,6 @@ namespace VulnCrawler ...@@ -30,7 +30,6 @@ namespace VulnCrawler
30 } 30 }
31 foreach (var commit in commits) { 31 foreach (var commit in commits) {
32 // 커밋 메시지 32 // 커밋 메시지
33 -
34 count++; 33 count++;
35 double per = ((double)count / (double)totalCount) * 100; 34 double per = ((double)count / (double)totalCount) * 100;
36 35
...@@ -46,23 +45,35 @@ namespace VulnCrawler ...@@ -46,23 +45,35 @@ namespace VulnCrawler
46 string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}"; 45 string commitUrl = $"{crawler.PushUrl}/commit/{commit.Sha}";
47 46
48 foreach (var parent in commit.Parents) { 47 foreach (var parent in commit.Parents) {
49 -
50 try 48 try
51 { 49 {
50 +
51 +
52 // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴 52 // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴
53 var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree); 53 var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree);
54 +
54 // 패치 엔트리 파일 배열 중에 파일 확장자가 .py인 것만 가져옴 55 // 패치 엔트리 파일 배열 중에 파일 확장자가 .py인 것만 가져옴
55 // (실질적인 코드 변경 커밋만 보기 위해서) 56 // (실질적인 코드 변경 커밋만 보기 위해서)
56 var entrys = crawler.GetPatchEntryChanges(patch); 57 var entrys = crawler.GetPatchEntryChanges(patch);
58 + if (entrys.Count() > 100)
59 + {
60 + // continue;
61 + }
57 /* C:\VulnC\linux 라면 linux만 뽑아서 repoName에 저장 */ 62 /* C:\VulnC\linux 라면 linux만 뽑아서 repoName에 저장 */
58 var dsp = dirPath.Split(Path.DirectorySeparatorChar); 63 var dsp = dirPath.Split(Path.DirectorySeparatorChar);
59 string repoName = dsp[dsp.Length - 1]; 64 string repoName = dsp[dsp.Length - 1];
60 // 현재 커밋에 대한 패치 엔트리 배열을 출력함 65 // 현재 커밋에 대한 패치 엔트리 배열을 출력함
61 PrintPatchEntrys(entrys, crawler, message, cve, repoName, commitUrl); 66 PrintPatchEntrys(entrys, crawler, message, cve, repoName, commitUrl);
62 // Console.ReadLine(); 67 // Console.ReadLine();
68 + break;
69 +
70 + }
71 + catch(Exception e)
72 + {
73 + break;
74 + //Console.WriteLine(e.ToString());
75 + //Console.ReadLine();
63 } 76 }
64 - catch(Exception)
65 - { }
66 } 77 }
67 } 78 }
68 } 79 }
...@@ -133,7 +144,7 @@ namespace VulnCrawler ...@@ -133,7 +144,7 @@ namespace VulnCrawler
133 #endregion 144 #endregion
134 145
135 } 146 }
136 - catch (Exception e) 147 + catch (Exception)
137 { 148 {
138 continue; 149 continue;
139 } 150 }
......
...@@ -9,22 +9,116 @@ using System.Text; ...@@ -9,22 +9,116 @@ using System.Text;
9 using System.Text.RegularExpressions; 9 using System.Text.RegularExpressions;
10 using System.Threading.Tasks; 10 using System.Threading.Tasks;
11 using VulnCrawler; 11 using VulnCrawler;
12 +using Newtonsoft.Json;
13 +using Newtonsoft.Json.Linq;
14 +
12 15
13 namespace VulnUserCodeAnalyzer 16 namespace VulnUserCodeAnalyzer
14 { 17 {
18 + public class CVE
19 + {
20 + public string Type { get; set; }
21 + public int Year { get; set; }
22 + //public string UserName { get; set; }
23 + public string Code { get; set; }
24 + public DateTime Publish_Date { get; set; }
25 + public DateTime Update_Date { get; set; }
26 + public string Detail { get; set; }
27 + //public string FileName { get; set; }
28 + //public string FuncNameBase64 { get; set; }
29 + //public string Url { get; set; }
30 + public double Level { get; set; }
31 + }
32 + public static class CVE_JSON
33 + {
34 + /// <summary>
35 + /// CVE 테이블
36 + /// </summary>
37 + public static Dictionary<int, Dictionary<string, CVE>> CveDict { get; set; }
38 + static CVE_JSON()
39 + {
40 + CveDict = new Dictionary<int, Dictionary<string, CVE>>();
41 + }
42 + public static void AutoLoad()
43 + {
44 + var dir = new DirectoryInfo(@"c:\CVE");
45 +
46 + foreach (var json in dir.EnumerateFiles("*.json"))
47 + {
48 + var match = Regex.Match(json.Name, @"(20\d\d)");
49 + if (!match.Success)
50 + {
51 + continue;
52 + }
53 + int year = int.Parse(match.Value);
54 +
55 + if (CveDict.ContainsKey(year))
56 + {
57 + continue;
58 + }
59 + var dict = LoadCveJson(int.Parse(match.Value));
60 + CveDict.Add(year, dict);
61 +
62 + Console.WriteLine($"cve 로드 완료 {year}, 개수 : {CveDict[year].Count}");
63 +
64 + }
65 + }
66 +
67 + /// <summary>
68 + /// CVE 정보 수집
69 + /// </summary>
70 + /// <param name="year"></param>
71 + /// <returns></returns>
72 + private static Dictionary<string, CVE> LoadCveJson(int year)
73 + {
74 + string json = File.ReadAllText($@"C:\CVE\{year}.json");
75 + JObject jobj = JObject.Parse(json);
76 + var cveDict = jobj["CVE_Items"].ToDictionary(t => t["cve"]["CVE_data_meta"]["ID"].ToString(), t =>
77 + {
78 + var vendor_data = t["cve"]["affects"]["vendor"]["vendor_data"] as JArray;
79 + string vendor_name = "NULL";
80 + if (vendor_data.Count > 0)
81 + {
82 + vendor_name = vendor_data.First()["vendor_name"].ToString();
83 + }
84 + var description_data = t["cve"]["description"]["description_data"] as JArray;
85 + string description = "NULL";
86 + if (description_data.Count > 0)
87 + {
88 + description = description_data.First()["value"].ToString();
89 + }
90 + double level = 0;
91 + var impact = t["impact"];
92 + if (impact.HasValues)
93 + {
94 + level = Double.Parse(impact["baseMetricV2"]["cvssV2"]["baseScore"].ToString());
95 + }
96 + return new CVE
97 + {
98 + Code = t["cve"]["CVE_data_meta"]["ID"].ToString(),
99 + Type = vendor_name,
100 + Detail = description,
101 + Year = year,
102 + Publish_Date = DateTime.Parse(t["publishedDate"].ToString()),
103 + Update_Date = DateTime.Parse(t["lastModifiedDate"].ToString()),
104 + Level = level,
105 + };
106 + });
107 + return cveDict;
108 + }
109 + }
15 class Program 110 class Program
16 { 111 {
17 static void Main(string[] args) 112 static void Main(string[] args)
18 { 113 {
19 - var crawler = new VulnC();
20 - //var bytes = Convert.FromBase64String("dgBvAGkAZAAgAGsAdgBtAF8AbQBtAHUAXwBuAGUAdwBfAGMAcgAzACgAcwB0AHIAdQBjAHQAIABrAHYAbQBfAHYAYwBwAHUAIAAqAHYAYwBwAHUAKQANAAoAewANAAoACQBtAG0AdQBfAGYAcgBlAGUAXwByAG8AbwB0AHMAKAB2AGMAcAB1ACkAOwANAAoAfQANAAoA");
21 - //var str = Encoding.Unicode.GetString(bytes);
22 114
23 - //Console.WriteLine(str); 115 + /* 연도별 CVE JSON 파일 로드 */
24 - //Console.WriteLine(crawler.Abstract(str, new Dictionary<string, string>(), new Dictionary<string, string>())); 116 + //CVE_JSON.AutoLoad();
25 - //Console.ReadLine();
26 117
27 - // default usage 118 + /* 크롤러 타입 */
119 + var crawler = new VulnC();
120 +
121 + /* 매칭을 위한 자료구조 Bloom Filter */
28 int capacity = 50000000; 122 int capacity = 50000000;
29 var filter = new Filter<string>(capacity); 123 var filter = new Filter<string>(capacity);
30 124
...@@ -32,10 +126,8 @@ namespace VulnUserCodeAnalyzer ...@@ -32,10 +126,8 @@ namespace VulnUserCodeAnalyzer
32 string txt = File.ReadAllText(@"Account.xml"); 126 string txt = File.ReadAllText(@"Account.xml");
33 // string xml = aes.AESDecrypt128(txt, key); 127 // string xml = aes.AESDecrypt128(txt, key);
34 string xml = txt; 128 string xml = txt;
35 -
36 AWS.LoadAccount(xml); 129 AWS.LoadAccount(xml);
37 AWS.Account account = AWS.account; 130 AWS.Account account = AWS.account;
38 -
39 /* AWS 정보 출력 */ 131 /* AWS 정보 출력 */
40 Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}"); 132 Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
41 try 133 try
...@@ -48,22 +140,34 @@ namespace VulnUserCodeAnalyzer ...@@ -48,22 +140,34 @@ namespace VulnUserCodeAnalyzer
48 Console.WriteLine($"접속 에러 :: {e.ToString()}"); 140 Console.WriteLine($"접속 에러 :: {e.ToString()}");
49 return; 141 return;
50 } 142 }
51 -
52 /* AWS 연결 여부 확인 */ 143 /* AWS 연결 여부 확인 */
53 if (VulnRDS.Conn.State == System.Data.ConnectionState.Open) 144 if (VulnRDS.Conn.State == System.Data.ConnectionState.Open)
54 { 145 {
55 Console.WriteLine("접속 성공"); 146 Console.WriteLine("접속 성공");
56 -
57 } 147 }
58 else 148 else
59 { 149 {
60 Console.WriteLine("연결 실패"); 150 Console.WriteLine("연결 실패");
61 return; 151 return;
62 } 152 }
153 +
154 + var reposits = VulnRDS.SelectAllReposit();
155 +
156 + foreach (var (userName, repository) in reposits)
157 + {
158 + Console.WriteLine($"{userName}, {repository}");
159 + }
160 +
161 + Console.ReadLine();
162 +
163 + /* hashDict = 사용된 사용자 함수 정보 */
63 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>(); 164 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
165 + /* 경과 시간 체크 */
64 Stopwatch stopwatch = new Stopwatch(); 166 Stopwatch stopwatch = new Stopwatch();
65 stopwatch.Start(); 167 stopwatch.Start();
66 - DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code"); 168 + DirectoryInfo dirInfo = new DirectoryInfo(@"C:\code");
169 +
170 + /* 모든 .c 파일 탐색 */
67 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories); 171 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
68 int totalFileCount = codeFiles.Count(); 172 int totalFileCount = codeFiles.Count();
69 int count = 0; 173 int count = 0;
...@@ -72,13 +176,18 @@ namespace VulnUserCodeAnalyzer ...@@ -72,13 +176,18 @@ namespace VulnUserCodeAnalyzer
72 Console.WriteLine(codeFile.FullName); 176 Console.WriteLine(codeFile.FullName);
73 using (var reader = codeFile.OpenText()) 177 using (var reader = codeFile.OpenText())
74 { 178 {
179 + /* 사용자 코드를 함수별로 나눔 */
75 var dict = crawler.CrawlUserCode(reader); 180 var dict = crawler.CrawlUserCode(reader);
76 foreach (var item in dict) 181 foreach (var item in dict)
77 { 182 {
183 + /* hashDict의 키와 item.key는 함수 블록의 코드 길이 */
78 if (!hashDict.ContainsKey(item.Key)) 184 if (!hashDict.ContainsKey(item.Key))
79 { 185 {
80 hashDict[item.Key] = new HashSet<VulnAbstractCrawler.UserBlock>(); 186 hashDict[item.Key] = new HashSet<VulnAbstractCrawler.UserBlock>();
81 } 187 }
188 + /* item.Value는 각 코드 길이 마다의 블록 정보
189 + * Bloom Filter에 코드 블록 해쉬값 기록
190 + */
82 foreach (var hash in item.Value) 191 foreach (var hash in item.Value)
83 { 192 {
84 hash.Path = codeFile.FullName; 193 hash.Path = codeFile.FullName;
...@@ -88,50 +197,69 @@ namespace VulnUserCodeAnalyzer ...@@ -88,50 +197,69 @@ namespace VulnUserCodeAnalyzer
88 } 197 }
89 count++; 198 count++;
90 double per = ((double)count / (double)totalFileCount) * 100; 199 double per = ((double)count / (double)totalFileCount) * 100;
91 - Console.Clear();
92 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}"); 200 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}");
93 - if (count > 100)
94 - {
95 - break;
96 - }
97 } 201 }
98 } 202 }
99 var findBlocks = new Queue<VulnAbstractCrawler.UserBlock>(); 203 var findBlocks = new Queue<VulnAbstractCrawler.UserBlock>();
100 var vulnDict = new Dictionary<string, IEnumerable<VulnRDS._Vuln>>(); 204 var vulnDict = new Dictionary<string, IEnumerable<VulnRDS._Vuln>>();
101 foreach (var set in hashDict) 205 foreach (var set in hashDict)
102 { 206 {
207 + /* 사용자 코드의 길이 마다 DB로 부터 같은 길이의 CVE 레코드 목록 가져옴 */
103 var cveList = VulnRDS.SelectVulnbyLen(set.Key).Select(v => v.Cve).Distinct(); 208 var cveList = VulnRDS.SelectVulnbyLen(set.Key).Select(v => v.Cve).Distinct();
104 foreach (var cve in cveList) 209 foreach (var cve in cveList)
105 { 210 {
106 if (!vulnDict.ContainsKey(cve)) 211 if (!vulnDict.ContainsKey(cve))
107 { 212 {
108 vulnDict[cve] = new HashSet<VulnRDS._Vuln>(); 213 vulnDict[cve] = new HashSet<VulnRDS._Vuln>();
109 - // SQL CVE 목록 가져와야 함 214 + var vulnHashSet = vulnDict[cve] as HashSet<VulnRDS._Vuln>;
110 - // 가져와서 각 CVE 마다 vulnDict에 추가 215 + /* 같은 길이의 CVE에서 또 같은 종류의 CVE 레코드 목록 가져옴
216 + * 같은 종류의 CVE 레코드들이 사용자 코드에서 모두 포함되어야
217 + * CVE를 가지고 있다고 인정하는 프로그램 정책 때문
218 + */
219 + var searchedCveHashList = VulnRDS.SelectVulnbyCve(cve);
220 + Console.WriteLine($"cve:{cve}, {searchedCveHashList.Count()}개 가져옴");
221 + foreach (var s in searchedCveHashList)
222 + {
223 + vulnHashSet.Add(s);
224 + }
225 +
111 } 226 }
112 } 227 }
113 } 228 }
114 - 229 + var findCveDict = new Dictionary<string, List<VulnAbstractCrawler.UserBlock>>();
230 + var findCveList = new HashSet<string>();
231 + /* 본격적인 취약점 매칭 부분 */
115 foreach (var vulnSet in vulnDict) 232 foreach (var vulnSet in vulnDict)
116 { 233 {
117 - Console.WriteLine($"-----cve:{vulnSet.Key}"); 234 + //Console.WriteLine($"-----cve:{vulnSet.Key}");
118 bool match = false; 235 bool match = false;
119 foreach (var vuln in vulnSet.Value) 236 foreach (var vuln in vulnSet.Value)
120 { 237 {
238 + /* 사용자 코드 해쉬 저장해논 bloom filter에 취약점 레코드 해쉬값들이 포함되는지 확인
239 + * 포함이 된다는 건 해당 취약점 레코드가 사용자 코드에도 있다는 뜻(취약점)
240 + * 같은 종류의 CVE 레코드가 전부 필터에 포함된다면 취약점으로 판단한다.
241 + */
121 if (filter.Contains(vuln.BlockHash)) 242 if (filter.Contains(vuln.BlockHash))
122 { 243 {
123 - Console.WriteLine($"필터 확인 : {vuln.BlockHash}");
124 if (hashDict.ContainsKey(vuln.LenFunc)) 244 if (hashDict.ContainsKey(vuln.LenFunc))
125 { 245 {
246 + /* Bloom Filter는 아쉽게도 포함 여부만 알 수 있기에
247 + * 포함되었음을 알았다면 검색해서 정보를 구한다. */
126 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash); 248 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
127 if (userBlock == null) 249 if (userBlock == null)
128 { 250 {
129 - Console.WriteLine("userBlock이 비어있습니다.");
130 continue; 251 continue;
131 } 252 }
132 - Console.WriteLine($"CVE:{vuln.Cve}, {userBlock.FuncName}, 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}"); 253 + /* 해당 유저 블록을 임시 저장한다.
254 + * 밑에서 블록 정보를 DB로 전송하기 위해서다.
255 + */
256 + if (!findCveDict.ContainsKey(vuln.Cve))
257 + {
258 + findCveDict[vuln.Cve] = new List<VulnAbstractCrawler.UserBlock>();
259 + }
260 + userBlock.Url = vuln.Url;
261 + findCveDict[vuln.Cve].Add(userBlock);
133 match = true; 262 match = true;
134 - findBlocks.Enqueue(userBlock);
135 } 263 }
136 } 264 }
137 else 265 else
...@@ -140,45 +268,97 @@ namespace VulnUserCodeAnalyzer ...@@ -140,45 +268,97 @@ namespace VulnUserCodeAnalyzer
140 break; 268 break;
141 } 269 }
142 } 270 }
271 + /* 취약점 레코드가 전부 있어야 CVE 찾음 인정 */
143 if (match) 272 if (match)
144 { 273 {
145 Console.WriteLine($"CVE 찾음 {vulnSet.Key}"); 274 Console.WriteLine($"CVE 찾음 {vulnSet.Key}");
275 + /* 찾았으면 cve값을 기록함 밑에서 찾은 cve 정보 전송하기 위해 */
276 + findCveList.Add(vulnSet.Key);
277 + }
278 + else
279 + {
280 + Console.WriteLine("없음");
146 } 281 }
147 } 282 }
148 -
149 stopwatch.Stop(); 283 stopwatch.Stop();
150 - 284 + /* 매칭 끝 후처리 (출력, DB 전송 등) */
151 - 285 + var hours = stopwatch.Elapsed.Hours;
152 - 286 + var minutes = stopwatch.Elapsed.Minutes;
153 - 287 + var seconds = stopwatch.Elapsed.Seconds;
154 - var hours = stopwatch.Elapsed.TotalHours;
155 - var minutes = stopwatch.Elapsed.TotalMinutes;
156 - var seconds = stopwatch.Elapsed.TotalSeconds;
157 -
158 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}"); 288 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
289 + Console.WriteLine($"찾은 CVE 개수 : {findCveList.Count}");
290 + var yearMatch = new Regex(@"CVE-(\d{4})-(\d+)");
291 + foreach (var cve in findCveList)
292 + {
293 + Console.WriteLine(cve);
294 + var c = yearMatch.Match(cve);
295 + int year = int.Parse(c.Groups[1].Value);
296 + if (!CVE_JSON.CveDict.ContainsKey(year))
297 + {
298 + continue;
299 + }
300 + if (!CVE_JSON.CveDict[year].ContainsKey(cve))
301 + {
302 + continue;
303 + }
304 + var data = CVE_JSON.CveDict[year][cve];
159 305
160 - 306 + /* 취약점 타입 분류 */
161 - // 블룸 필터 테스트 307 + string type = "NORMAL";
162 - //while(true) 308 + if (data.Detail.IndexOf("overflow", StringComparison.CurrentCultureIgnoreCase) > 0)
163 - //{ 309 + {
164 - // string key = Console.ReadLine(); 310 + type = "OVERFLOW";
165 - // if (key == "-1") 311 + }
166 - // { 312 + else if (data.Detail.IndexOf("xss", StringComparison.CurrentCultureIgnoreCase) > 0)
167 - // break; 313 + {
168 - // } 314 + type = "XSS";
169 - // if (filter.Contains(key)) 315 + }
170 - // { 316 + else if (data.Detail.IndexOf("injection", StringComparison.CurrentCultureIgnoreCase) > 0)
171 - // Console.WriteLine("포함"); 317 + {
172 - // } 318 + type = "SQLINJECTION";
173 - // else 319 + }
174 - // { 320 + else if (data.Detail.IndexOf("dos", StringComparison.CurrentCultureIgnoreCase) > 0)
175 - // Console.WriteLine("없음"); 321 + {
176 - // } 322 + type = "DOS";
177 - 323 + }
324 + else if (data.Detail.IndexOf("Memory", StringComparison.CurrentCultureIgnoreCase) > 0)
325 + {
326 + type = "MEMORY";
327 + }
328 + else if (data.Detail.IndexOf("CSRF", StringComparison.CurrentCultureIgnoreCase) > 0)
329 + {
330 + type = "CSRF";
331 + }
332 + else if (data.Detail.IndexOf("inclusion", StringComparison.CurrentCultureIgnoreCase) > 0)
333 + {
334 + type = "FILEINCLUSION";
335 + }
336 + else if (data.Detail.IndexOf("EXCUTE", StringComparison.CurrentCultureIgnoreCase) > 0)
337 + {
338 + type = "EXCUTE";
339 + }
178 340
179 - //} 341 + var urlBytes = Convert.FromBase64String(findCveDict[cve].FirstOrDefault().Url);
180 - 342 + string url = Encoding.Unicode.GetString(urlBytes);
181 343
344 + /* DB 전송 */
345 + VulnRDS.InsertVulnDetail(new VulnRDS.Vuln_detail
346 + {
347 + CveName = data.Code,
348 + Type = type,
349 + Level = data.Level.ToString(),
350 + Year = data.Year.ToString(),
351 + CveDetail = data.Detail,
352 + Publish_date = data.Publish_Date.ToString("yyyy-MM-dd"),
353 + Update_date = data.Update_Date.ToString("yyyy-MM-dd"),
354 + UserName = "samsung",
355 + Url = url,
356 + FileName = findCveDict[cve].FirstOrDefault().Path.Replace(@"C:\code", ""),
357 + FuncName = findCveDict[cve].FirstOrDefault().FuncName,
358 + Product = data.Type,
359 + });
360 + Console.WriteLine("추가 완료");
361 + }
182 } 362 }
183 } 363 }
184 364
......
...@@ -33,8 +33,14 @@ ...@@ -33,8 +33,14 @@
33 </PropertyGroup> 33 </PropertyGroup>
34 <ItemGroup> 34 <ItemGroup>
35 <Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" /> 35 <Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" />
36 + <Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
37 + <HintPath>..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
38 + </Reference>
36 <Reference Include="System" /> 39 <Reference Include="System" />
37 <Reference Include="System.Core" /> 40 <Reference Include="System.Core" />
41 + <Reference Include="System.ValueTuple, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
42 + <HintPath>..\packages\System.ValueTuple.4.5.0\lib\net461\System.ValueTuple.dll</HintPath>
43 + </Reference>
38 <Reference Include="System.Xml.Linq" /> 44 <Reference Include="System.Xml.Linq" />
39 <Reference Include="System.Data.DataSetExtensions" /> 45 <Reference Include="System.Data.DataSetExtensions" />
40 <Reference Include="Microsoft.CSharp" /> 46 <Reference Include="Microsoft.CSharp" />
...@@ -49,6 +55,7 @@ ...@@ -49,6 +55,7 @@
49 </ItemGroup> 55 </ItemGroup>
50 <ItemGroup> 56 <ItemGroup>
51 <None Include="App.config" /> 57 <None Include="App.config" />
58 + <None Include="packages.config" />
52 </ItemGroup> 59 </ItemGroup>
53 <ItemGroup> 60 <ItemGroup>
54 <ProjectReference Include="..\VulnCrawler\VulnCrawler.csproj"> 61 <ProjectReference Include="..\VulnCrawler\VulnCrawler.csproj">
......
1 +<?xml version="1.0" encoding="utf-8"?>
2 +<packages>
3 + <package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" />
4 + <package id="System.ValueTuple" version="4.5.0" targetFramework="net461" />
5 +</packages>
...\ No newline at end of file ...\ No newline at end of file