노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
이지윤
0e3e7355 2 parents 17a22d4c bcdb9649

Merge branch 'master' of https://github.com/yhackerbv/VulnNottiProject

Inline Side-by-side
Showing 5 changed files with 216 additions and 121 deletions
...@@ -65,10 +65,6 @@ namespace VulnCrawler ...@@ -65,10 +65,6 @@ namespace VulnCrawler
65 65
66 /* 메인 동작 함수 */ 66 /* 메인 동작 함수 */
67 public static void Run() { 67 public static void Run() {
68 -
69 -
70 -
71 -
72 // Repository 폴더들이 있는 주소를 지정하면 하위 폴더 목록을 가져옴(Repository 목록) 68 // Repository 폴더들이 있는 주소를 지정하면 하위 폴더 목록을 가져옴(Repository 목록)
73 Regex.CacheSize = 50; 69 Regex.CacheSize = 50;
74 70
...@@ -88,35 +84,6 @@ namespace VulnCrawler ...@@ -88,35 +84,6 @@ namespace VulnCrawler
88 { 84 {
89 continue; 85 continue;
90 } 86 }
91 - //var crawler = new VulnC();
92 - //crawler.Init(directory);
93 -
94 - //Tree commitTree1 = crawler.Repository.Lookup<Commit>("e589db7a6a9c8f1557007f2cc765ee28ad7a1edd").Tree;
95 - ////Tree parentTree1 = crawler.Repository.Lookup<Commit>("344ba37bdc299660e1b1693b6999e5fe116893e1").Tree;
96 -
97 - //Commit commit = crawler.Repository.Lookup<Commit>("e589db7a6a9c8f1557007f2cc765ee28ad7a1edd");
98 -
99 - //foreach (var parent in commit.Parents)
100 - //{
101 - // Console.WriteLine($"Parent ID:{parent.Sha}");
102 - // Tree commitTree = commit.Tree;
103 - // Tree parentTree = parent.Tree;
104 - // var patch = crawler.Repository.Diff.Compare<Patch>(commitTree, parentTree.);
105 - // foreach (var item in patch.Where(p => p.OldPath.EndsWith(".c")))
106 - // {
107 -
108 - // Console.WriteLine(item.Status);
109 - // Console.WriteLine(item.Path);
110 - // Console.WriteLine(item.Patch);
111 - // }
112 - // Console.ReadLine();
113 -
114 - //}
115 -
116 -
117 - ////Console.WriteLine(patch.Content);
118 - //Console.ReadLine();
119 -
120 // 템플릿 패턴화 T : VulnAbstractCrawler 87 // 템플릿 패턴화 T : VulnAbstractCrawler
121 VulnWorker.Run<VulnC>(directory); 88 VulnWorker.Run<VulnC>(directory);
122 } 89 }
......
...@@ -31,7 +31,7 @@ namespace VulnCrawler ...@@ -31,7 +31,7 @@ namespace VulnCrawler
31 public string FuncName { get; set; } 31 public string FuncName { get; set; }
32 public string Hash { get; set; } 32 public string Hash { get; set; }
33 public string Path { get; set; } 33 public string Path { get; set; }
34 - 34 + public string Url { get; set; }
35 public override bool Equals(object obj) 35 public override bool Equals(object obj)
36 { 36 {
37 var block = obj as UserBlock; 37 var block = obj as UserBlock;
...@@ -131,7 +131,7 @@ namespace VulnCrawler ...@@ -131,7 +131,7 @@ namespace VulnCrawler
131 /// <summary> 131 /// <summary>
132 /// 커밋에서 검색할 정규식 문자열 132 /// 커밋에서 검색할 정규식 문자열
133 /// </summary> 133 /// </summary>
134 - public string SearchCommitPattern => @"CVE[ -](\d{4})[ -](\d{4,})"; 134 + public string SearchCommitPattern => @"CVE[ -](201[5-8])[ -](\d{4,})";
135 /// <summary> 135 /// <summary>
136 /// 패치 코드에서 함수 찾을 정규식 패턴 문자열 136 /// 패치 코드에서 함수 찾을 정규식 패턴 문자열
137 /// </summary> 137 /// </summary>
......
...@@ -891,15 +891,10 @@ namespace VulnCrawler ...@@ -891,15 +891,10 @@ namespace VulnCrawler
891 var regex2 = new Regex(stringPattern, RegexOptions.Compiled); 891 var regex2 = new Regex(stringPattern, RegexOptions.Compiled);
892 var regex3 = new Regex(commentPattern2, RegexOptions.Compiled); 892 var regex3 = new Regex(commentPattern2, RegexOptions.Compiled);
893 var regex4 = new Regex(commentPattern, RegexOptions.Compiled); 893 var regex4 = new Regex(commentPattern, RegexOptions.Compiled);
894 -
895 bool found3 = false; 894 bool found3 = false;
896 -
897 bool com = false; 895 bool com = false;
898 -
899 -
900 while (!reader.EndOfStream) 896 while (!reader.EndOfStream)
901 { 897 {
902 -
903 string line = reader.ReadLine(); 898 string line = reader.ReadLine();
904 string trim = line.Trim(); 899 string trim = line.Trim();
905 if (commentLine) 900 if (commentLine)
...@@ -915,7 +910,6 @@ namespace VulnCrawler ...@@ -915,7 +910,6 @@ namespace VulnCrawler
915 continue; 910 continue;
916 } 911 }
917 } 912 }
918 -
919 // /* ~ 패턴 913 // /* ~ 패턴
920 if (regex3.IsMatch(trim)) 914 if (regex3.IsMatch(trim))
921 { 915 {
...@@ -941,20 +935,18 @@ namespace VulnCrawler ...@@ -941,20 +935,18 @@ namespace VulnCrawler
941 if (found3) 935 if (found3)
942 { 936 {
943 string obStr = oldBuilder.ToString(); 937 string obStr = oldBuilder.ToString();
944 - Console.WriteLine(obStr); 938 + //Console.WriteLine(obStr);
945 -
946 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>()); 939 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>());
947 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr); 940 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
948 string absObStrBase64 = Convert.ToBase64String(obStrBytes); 941 string absObStrBase64 = Convert.ToBase64String(obStrBytes);
949 - 942 + // Console.WriteLine(obStr);
950 - Console.WriteLine(obStr); 943 + //Console.WriteLine("HASH: " + MD5HashFunc(obStr));
944 + //Console.WriteLine(absObStrBase64);
951 if (!dict.ContainsKey(absObStrBase64.Length)) 945 if (!dict.ContainsKey(absObStrBase64.Length))
952 { 946 {
953 dict[absObStrBase64.Length] = new HashSet<UserBlock>(); 947 dict[absObStrBase64.Length] = new HashSet<UserBlock>();
954 } 948 }
955 -
956 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray()); 949 string funcName = new string(oldBuilder.ToString().TakeWhile(c => c != '{').ToArray());
957 -
958 (dict[absObStrBase64.Length] as HashSet<UserBlock>).Add(new UserBlock 950 (dict[absObStrBase64.Length] as HashSet<UserBlock>).Add(new UserBlock
959 { 951 {
960 Hash = MD5HashFunc(absObStrBase64), 952 Hash = MD5HashFunc(absObStrBase64),
...@@ -1090,11 +1082,11 @@ namespace VulnCrawler ...@@ -1090,11 +1082,11 @@ namespace VulnCrawler
1090 if (found3) 1082 if (found3)
1091 { 1083 {
1092 string obStr = oldBuilder.ToString(); 1084 string obStr = oldBuilder.ToString();
1093 - Console.WriteLine(obStr); 1085 + // Console.WriteLine(obStr);
1094 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>()); 1086 obStr = Abstract(obStr, new Dictionary<string, string>(), new Dictionary<string, string>());
1095 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr); 1087 byte[] obStrBytes = Encoding.Unicode.GetBytes(obStr);
1096 string absObStrBase64 = Convert.ToBase64String(obStrBytes); 1088 string absObStrBase64 = Convert.ToBase64String(obStrBytes);
1097 - Console.WriteLine(obStr); 1089 + // Console.WriteLine(obStr);
1098 if (!dict.ContainsKey(absObStrBase64.Length)) 1090 if (!dict.ContainsKey(absObStrBase64.Length))
1099 { 1091 {
1100 dict[absObStrBase64.Length] = new HashSet<UserBlock>(); 1092 dict[absObStrBase64.Length] = new HashSet<UserBlock>();
......
...@@ -67,6 +67,7 @@ namespace VulnCrawler ...@@ -67,6 +67,7 @@ namespace VulnCrawler
67 public string FileName { get; set; } = "NULL"; /* FileName */ 67 public string FileName { get; set; } = "NULL"; /* FileName */
68 public string FuncName { get; set; } = "NULL"; /* funcName */ 68 public string FuncName { get; set; } = "NULL"; /* funcName */
69 public string Url { get; set; } = "NULL"; /* Url */ 69 public string Url { get; set; } = "NULL"; /* Url */
70 + public string Product { get; set; }
70 71
71 } 72 }
72 //connect 73 //connect
...@@ -242,7 +243,7 @@ namespace VulnCrawler ...@@ -242,7 +243,7 @@ namespace VulnCrawler
242 { 243 {
243 Connection = Conn, 244 Connection = Conn,
244 //db에 추가 245 //db에 추가
245 - CommandText = "INSERT INTO vulnDetail(type, year, level, userName, cveName, publish_date,update_date, cveDetail,fileName, funcName, url) VALUES(@type, @year, @level, @userName, @cveName, @publish_date,@update_date, @cveDetail,@fileName, @funcName,@url)" 246 + CommandText = "INSERT INTO vulnDetail(type, year, level, userName, cveName, publish_date,update_date, cveDetail,fileName, funcName, url, product) VALUES(@type, @year, @level, @userName, @cveName, @publish_date,@update_date, @cveDetail,@fileName, @funcName,@url,@product)"
246 }; 247 };
247 cmd.Parameters.AddWithValue("@type", $"{vuln.Type}"); 248 cmd.Parameters.AddWithValue("@type", $"{vuln.Type}");
248 cmd.Parameters.AddWithValue("@year", $"{vuln.Year}"); 249 cmd.Parameters.AddWithValue("@year", $"{vuln.Year}");
...@@ -255,16 +256,16 @@ namespace VulnCrawler ...@@ -255,16 +256,16 @@ namespace VulnCrawler
255 cmd.Parameters.AddWithValue("@fileName", $"{vuln.FileName}"); 256 cmd.Parameters.AddWithValue("@fileName", $"{vuln.FileName}");
256 cmd.Parameters.AddWithValue("@funcName", $"{vuln.FuncName}"); 257 cmd.Parameters.AddWithValue("@funcName", $"{vuln.FuncName}");
257 cmd.Parameters.AddWithValue("@url", $"{vuln.Url}"); 258 cmd.Parameters.AddWithValue("@url", $"{vuln.Url}");
258 - 259 + cmd.Parameters.AddWithValue("@product", $"{vuln.Product}");
259 cmd.ExecuteNonQuery(); 260 cmd.ExecuteNonQuery();
260 //콘솔출력용 261 //콘솔출력용
261 sql = "INSERT INTO vulnDetail(type, year, level, userName, cveName, publish_date,update_date, cveDetail,fileName, funcName, url) " + 262 sql = "INSERT INTO vulnDetail(type, year, level, userName, cveName, publish_date,update_date, cveDetail,fileName, funcName, url) " +
262 $"VALUES({vuln.Type}, {vuln.Year}, {vuln.Level}, {vuln.UserName}, {vuln.CveName},{vuln.Publish_date}, {vuln.Update_date}, {vuln.CveDetail}, {vuln.FileName}, {vuln.FuncName}, {vuln.Url})"; 263 $"VALUES({vuln.Type}, {vuln.Year}, {vuln.Level}, {vuln.UserName}, {vuln.CveName},{vuln.Publish_date}, {vuln.Update_date}, {vuln.CveDetail}, {vuln.FileName}, {vuln.FuncName}, {vuln.Url})";
263 - Console.WriteLine(sql); 264 + // Console.WriteLine(sql);
264 } 265 }
265 catch (Exception e) 266 catch (Exception e)
266 { 267 {
267 - Console.WriteLine(e.ToString()); 268 + // Console.WriteLine(e.ToString());
268 string es = e.ToString(); 269 string es = e.ToString();
269 if (es.Contains("Connection must be valid and open")) 270 if (es.Contains("Connection must be valid and open"))
270 { 271 {
......
...@@ -15,26 +15,109 @@ using Newtonsoft.Json.Linq; ...@@ -15,26 +15,109 @@ using Newtonsoft.Json.Linq;
15 15
16 namespace VulnUserCodeAnalyzer 16 namespace VulnUserCodeAnalyzer
17 { 17 {
18 + public class CVE
19 + {
20 + public string Type { get; set; }
21 + public int Year { get; set; }
22 + //public string UserName { get; set; }
23 + public string Code { get; set; }
24 + public DateTime Publish_Date { get; set; }
25 + public DateTime Update_Date { get; set; }
26 + public string Detail { get; set; }
27 + //public string FileName { get; set; }
28 + //public string FuncNameBase64 { get; set; }
29 + //public string Url { get; set; }
30 + public double Level { get; set; }
31 + }
32 + public static class CVE_JSON
33 + {
34 + /// <summary>
35 + /// CVE 테이블
36 + /// </summary>
37 + public static Dictionary<int, Dictionary<string, CVE>> CveDict { get; set; }
38 + static CVE_JSON()
39 + {
40 + CveDict = new Dictionary<int, Dictionary<string, CVE>>();
41 + }
42 + public static void AutoLoad()
43 + {
44 + var dir = new DirectoryInfo(@"c:\CVE");
45 +
46 + foreach (var json in dir.EnumerateFiles("*.json"))
47 + {
48 + var match = Regex.Match(json.Name, @"(20\d\d)");
49 + if (!match.Success)
50 + {
51 + continue;
52 + }
53 + int year = int.Parse(match.Value);
54 +
55 + if (CveDict.ContainsKey(year))
56 + {
57 + continue;
58 + }
59 + var dict = LoadCveJson(int.Parse(match.Value));
60 + CveDict.Add(year, dict);
61 +
62 + Console.WriteLine($"cve 로드 완료 {year}, 개수 : {CveDict[year].Count}");
63 +
64 + }
65 + }
66 +
67 + /// <summary>
68 + /// CVE 정보 수집
69 + /// </summary>
70 + /// <param name="year"></param>
71 + /// <returns></returns>
72 + private static Dictionary<string, CVE> LoadCveJson(int year)
73 + {
74 + string json = File.ReadAllText($@"C:\CVE\{year}.json");
75 + JObject jobj = JObject.Parse(json);
76 + var cveDict = jobj["CVE_Items"].ToDictionary(t => t["cve"]["CVE_data_meta"]["ID"].ToString(), t =>
77 + {
78 + var vendor_data = t["cve"]["affects"]["vendor"]["vendor_data"] as JArray;
79 + string vendor_name = "NULL";
80 + if (vendor_data.Count > 0)
81 + {
82 + vendor_name = vendor_data.First()["vendor_name"].ToString();
83 + }
84 + var description_data = t["cve"]["description"]["description_data"] as JArray;
85 + string description = "NULL";
86 + if (description_data.Count > 0)
87 + {
88 + description = description_data.First()["value"].ToString();
89 + }
90 + double level = 0;
91 + var impact = t["impact"];
92 + if (impact.HasValues)
93 + {
94 + level = Double.Parse(impact["baseMetricV2"]["cvssV2"]["baseScore"].ToString());
95 + }
96 + return new CVE
97 + {
98 + Code = t["cve"]["CVE_data_meta"]["ID"].ToString(),
99 + Type = vendor_name,
100 + Detail = description,
101 + Year = year,
102 + Publish_Date = DateTime.Parse(t["publishedDate"].ToString()),
103 + Update_Date = DateTime.Parse(t["lastModifiedDate"].ToString()),
104 + Level = level,
105 + };
106 + });
107 + return cveDict;
108 + }
109 + }
18 class Program 110 class Program
19 { 111 {
20 static void Main(string[] args) 112 static void Main(string[] args)
21 { 113 {
22 - //string json = File.ReadAllText(@"C:\Users\haena\Downloads\cvelist-master\2018\5xxx\CVE-2018-5004.json"); 114 + /* 연도별 CVE JSON 파일 로드 */
23 - //JObject jobj = JObject.Parse(json); 115 + CVE_JSON.AutoLoad();
24 - //Console.WriteLine(jobj["CVE_data_meta"].ToString());
25 -
26 116
117 + /* 크롤러 타입 */
27 var crawler = new VulnC(); 118 var crawler = new VulnC();
28 - //var bytes = Convert.FromBase64String("cwB0AGEAdABpAGMAIABpAG4AdAAgAGMAaABhAGMAaABhADIAMABfAHAAbwBsAHkAMQAzADAANQBfAGMAaQBwAGgAZQByACgARQBWAFAAXwBDAEkAUABIAEUAUgBfAEMAVABYACAAKgBjAHQAeAAsACAAdQBuAHMAaQBnAG4AZQBkACAAYwBoAGEAcgAgACoAbwB1AHQALAANAAoAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAYwBvAG4AcwB0ACAAdQBuAHMAaQBnAG4AZQBkACAAYwBoAGEAcgAgACoAaQBuACwAIABzAGkAegBlAF8AdAAgAGwAZQBuACkADQAKAHsADQAKACAAIAAgACAARQBWAFAAXwBDAEgAQQBDAEgAQQBfAEEARQBBAEQAXwBDAFQAWAAgACoAYQBjAHQAeAAgAD0AIABhAGUAYQBkAF8AZABhAHQAYQAoAGMAdAB4ACkAOwANAAoAIAAgACAAIABzAGkAegBlAF8AdAAgAHIAZQBtACwAIABwAGwAZQBuACAAPQAgAGEAYwB0AHgALQA+AHQAbABzAF8AcABhAHkAbABvAGEAZABfAGwAZQBuAGcAdABoADsADQAKACAAIAAgACAAcwB0AGEAdABpAGMAIABjAG8AbgBzAHQAIAB1AG4AcwBpAGcAbgBlAGQAIABjAGgAYQByACAAegBlAHIAbwBbAFAATwBMAFkAMQAzADAANQBfAEIATABPAEMASwBfAFMASQBaAEUAXQAgAD0AIAB7ACAAMAAgAH0AOwANAAoAIAAgACAAIABpAGYAIAAoACEAYQBjAHQAeAAtAD4AbQBhAGMAXwBpAG4AaQB0AGUAZAApACAAewANAAoAIAAgACAAIAAgACAAIAAgAGEAYwB0AHgALQA+AGsAZQB5AC4AYwBvAHUAbgB0AGUAcgBbADAAXQAgAD0AIAAwADsADQAKACAAIAAgACAAIAAgACAAIABtAGUAbQBzAGUAdAAoAGEAYwB0AHgALQA+AGsAZQB5AC4AYgB1AGYALAAgADAALAAgAHMAaQB6AGUAbwBmACgAYQBjAHQAeAAtAD4AawBlAHkALgBiAHUAZgApACkAOwANAAoAIAAgACAAIAAgACAAIAAgAEMAaABhAEMAaABhADIAMABfAGMAdAByADMAMgAoAGEAYwB0AHgALQA+AGsAZQB5AC4AYgB1AGYALAAgAGEAYwB0AHgALQA+AGsAZQB5AC4AYgB1AGYALAAgAEMASABBAEMASABBAF8AQgBMAEsAXwBTAEkAWgBFACwADQAKACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIABhAGMAdAB4AC0APgBrAGUAeQAuAGsAZQB5AC4AZAAsACAAYQBjAHQAeAAtAD4AawBlAHkALgBjAG8AdQBuAHQAZQByACkAOwANAAoAIAAgACAAIAAgACAAIAAgAFAAbwBsAHkAMQAzADAANQBfAEkAbgBpAHQAKABQAE8ATABZADEAMwAwADUAXwBjAHQAeAAoAGEAYwB0AHgAKQAsACAAYQBjAHQAeAAtAD4AawBlAHkALgBiAHUAZgApADsADQAKACAAIAAgACAAIAAgACAAIABhAGMAdAB4AC0APgBrAGUAeQAuAGMAbwB1AG4AdABlAHIAWwAwAF0AIAA9ACAAMQA7AA0ACgAgACAAIAAgACAAIAAgACAAYQBjAHQAeAAtAD4AawBlAHkALgBwAGEAcgB0AGkAYQBsAF8AbABlAG4AIAA9ACAAMAA7AA0ACgAgACAAIAAgACAAIAAgACAAYQBjAHQAeAAtAD4AbABlAG4ALgBhAGEAZAAgAD0AIABhAGMAdAB4AC0APgBsAGUAbgAuAHQAZQB4AHQAIAA9ACAAMAA7AA0ACgAgACAAIAAgACAAIAAgACAAYQBjAHQAeAAtAD4AbQBhAGMAXwBpAG4AaQB0AGUAZAAgAD0AIAAxADsADQAKACAAIAAgACAAfQANAAoAIAAgACAAIAAgACAAIAAgACAAIAAgACAAUABvAGwAeQAxADMAMAA1AF8AVQBwAGQAYQB0AGUAKABQAE8ATABZADEAMwAwADUAXwBjAHQAeAAoAGEAYwB0AHgAKQAsACAAaQBuACwAIABsAGUAbgApADsADQAKACAAIAAgACAAIAAgACAAIAAgACAAIAAgAGEAYwB0AHgALQA+AGwAZQBuAC4AYQBhAGQAIAArAD0AIABsAGUAbgA7AA0ACgAgACAAIAAgACAAIAAgACAAIAAgACAAIABhAGMAdAB4AC0APgBhAGEAZAAgAD0AIAAxADsADQAKACAAIAAgACAAIAAgACAAIAAgACAAIAAgAHIAZQB0AHUAcgBuACAAbABlAG4AOwANAAoAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIABpAGYAIAAoACgAcgBlAG0AIAA9ACAAKABzAGkAegBlAF8AdAApAGEAYwB0AHgALQA+AGwAZQBuAC4AYQBhAGQAIAAlACAAUABPAEwAWQAxADMAMAA1AF8AQgBMAE8AQwBLAF8AUwBJAFoARQApACkADQAKACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIABQAG8AbAB5ADEAMwAwADUAXwBVAHAAZABhAHQAZQAoAFAATwBMAFkAMQAzADAANQBfAGMAdAB4ACgAYQBjAHQAeAApACwAIAB6AGUAcgBvACwADQAKACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgAFAATwBMAFkAMQAzADAANQBfAEIATABPAEMASwBfAFMASQBaAEUAIAAtACAAcgBlAG0AKQA7AA0ACgAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgACAAIAAgAGEAYwB0AHgALQA+AGEAYQBkACAAPQAgADAAOwANAAoAIAAgACAAIAAgACAAIAAgACAAIAAgACAAfQANAAoA");
29 - //var str = Encoding.Unicode.GetString(bytes);
30 119
31 - //Console.WriteLine(str); 120 + /* 매칭을 위한 자료구조 Bloom Filter */
32 - //var abs = crawler.Abstract(str, new Dictionary<string, string>(), new Dictionary<string, string>());
33 - //Console.WriteLine(abs);
34 - //Console.WriteLine(VulnAbstractCrawler.MD5HashFunc(abs));
35 - //Console.ReadLine();
36 -
37 - // default usage
38 int capacity = 50000000; 121 int capacity = 50000000;
39 var filter = new Filter<string>(capacity); 122 var filter = new Filter<string>(capacity);
40 123
...@@ -42,10 +125,8 @@ namespace VulnUserCodeAnalyzer ...@@ -42,10 +125,8 @@ namespace VulnUserCodeAnalyzer
42 string txt = File.ReadAllText(@"Account.xml"); 125 string txt = File.ReadAllText(@"Account.xml");
43 // string xml = aes.AESDecrypt128(txt, key); 126 // string xml = aes.AESDecrypt128(txt, key);
44 string xml = txt; 127 string xml = txt;
45 -
46 AWS.LoadAccount(xml); 128 AWS.LoadAccount(xml);
47 AWS.Account account = AWS.account; 129 AWS.Account account = AWS.account;
48 -
49 /* AWS 정보 출력 */ 130 /* AWS 정보 출력 */
50 Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}"); 131 Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
51 try 132 try
...@@ -58,64 +139,62 @@ namespace VulnUserCodeAnalyzer ...@@ -58,64 +139,62 @@ namespace VulnUserCodeAnalyzer
58 Console.WriteLine($"접속 에러 :: {e.ToString()}"); 139 Console.WriteLine($"접속 에러 :: {e.ToString()}");
59 return; 140 return;
60 } 141 }
61 -
62 /* AWS 연결 여부 확인 */ 142 /* AWS 연결 여부 확인 */
63 if (VulnRDS.Conn.State == System.Data.ConnectionState.Open) 143 if (VulnRDS.Conn.State == System.Data.ConnectionState.Open)
64 { 144 {
65 Console.WriteLine("접속 성공"); 145 Console.WriteLine("접속 성공");
66 -
67 } 146 }
68 else 147 else
69 { 148 {
70 Console.WriteLine("연결 실패"); 149 Console.WriteLine("연결 실패");
71 return; 150 return;
72 } 151 }
152 +
153 + /* hashDict = 사용된 사용자 함수 정보 */
73 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>(); 154 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
155 + /* 경과 시간 체크 */
74 Stopwatch stopwatch = new Stopwatch(); 156 Stopwatch stopwatch = new Stopwatch();
75 stopwatch.Start(); 157 stopwatch.Start();
76 DirectoryInfo dirInfo = new DirectoryInfo(@"C:\code"); 158 DirectoryInfo dirInfo = new DirectoryInfo(@"C:\code");
159 +
160 + /* 모든 .c 파일 탐색 */
77 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories); 161 var codeFiles = dirInfo.EnumerateFiles("*.c", SearchOption.AllDirectories);
78 int totalFileCount = codeFiles.Count(); 162 int totalFileCount = codeFiles.Count();
79 int count = 0; 163 int count = 0;
80 foreach (var codeFile in codeFiles) 164 foreach (var codeFile in codeFiles)
81 { 165 {
82 - // Process.Start(codeFile.FullName);
83 Console.WriteLine(codeFile.FullName); 166 Console.WriteLine(codeFile.FullName);
84 using (var reader = codeFile.OpenText()) 167 using (var reader = codeFile.OpenText())
85 { 168 {
169 + /* 사용자 코드를 함수별로 나눔 */
86 var dict = crawler.CrawlUserCode(reader); 170 var dict = crawler.CrawlUserCode(reader);
87 -
88 -
89 foreach (var item in dict) 171 foreach (var item in dict)
90 { 172 {
173 + /* hashDict의 키와 item.key는 함수 블록의 코드 길이 */
91 if (!hashDict.ContainsKey(item.Key)) 174 if (!hashDict.ContainsKey(item.Key))
92 { 175 {
93 hashDict[item.Key] = new HashSet<VulnAbstractCrawler.UserBlock>(); 176 hashDict[item.Key] = new HashSet<VulnAbstractCrawler.UserBlock>();
94 } 177 }
178 + /* item.Value는 각 코드 길이 마다의 블록 정보
179 + * Bloom Filter에 코드 블록 해쉬값 기록
180 + */
95 foreach (var hash in item.Value) 181 foreach (var hash in item.Value)
96 { 182 {
97 hash.Path = codeFile.FullName; 183 hash.Path = codeFile.FullName;
98 -
99 hashDict[item.Key].Add(hash); 184 hashDict[item.Key].Add(hash);
100 filter.Add(hash.Hash); 185 filter.Add(hash.Hash);
101 } 186 }
102 } 187 }
103 count++; 188 count++;
104 double per = ((double)count / (double)totalFileCount) * 100; 189 double per = ((double)count / (double)totalFileCount) * 100;
105 - //Console.Clear();
106 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}"); 190 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}");
107 - //if (count > 100)
108 - //{
109 - // break;
110 - //}
111 } 191 }
112 } 192 }
113 - // Console.ReadLine();
114 -
115 var findBlocks = new Queue<VulnAbstractCrawler.UserBlock>(); 193 var findBlocks = new Queue<VulnAbstractCrawler.UserBlock>();
116 var vulnDict = new Dictionary<string, IEnumerable<VulnRDS._Vuln>>(); 194 var vulnDict = new Dictionary<string, IEnumerable<VulnRDS._Vuln>>();
117 foreach (var set in hashDict) 195 foreach (var set in hashDict)
118 { 196 {
197 + /* 사용자 코드의 길이 마다 DB로 부터 같은 길이의 CVE 레코드 목록 가져옴 */
119 var cveList = VulnRDS.SelectVulnbyLen(set.Key).Select(v => v.Cve).Distinct(); 198 var cveList = VulnRDS.SelectVulnbyLen(set.Key).Select(v => v.Cve).Distinct();
120 foreach (var cve in cveList) 199 foreach (var cve in cveList)
121 { 200 {
...@@ -123,6 +202,10 @@ namespace VulnUserCodeAnalyzer ...@@ -123,6 +202,10 @@ namespace VulnUserCodeAnalyzer
123 { 202 {
124 vulnDict[cve] = new HashSet<VulnRDS._Vuln>(); 203 vulnDict[cve] = new HashSet<VulnRDS._Vuln>();
125 var vulnHashSet = vulnDict[cve] as HashSet<VulnRDS._Vuln>; 204 var vulnHashSet = vulnDict[cve] as HashSet<VulnRDS._Vuln>;
205 + /* 같은 길이의 CVE에서 또 같은 종류의 CVE 레코드 목록 가져옴
206 + * 같은 종류의 CVE 레코드들이 사용자 코드에서 모두 포함되어야
207 + * CVE를 가지고 있다고 인정하는 프로그램 정책 때문
208 + */
126 var searchedCveHashList = VulnRDS.SelectVulnbyCve(cve); 209 var searchedCveHashList = VulnRDS.SelectVulnbyCve(cve);
127 Console.WriteLine($"cve:{cve}, {searchedCveHashList.Count()}개 가져옴"); 210 Console.WriteLine($"cve:{cve}, {searchedCveHashList.Count()}개 가져옴");
128 foreach (var s in searchedCveHashList) 211 foreach (var s in searchedCveHashList)
...@@ -133,86 +216,138 @@ namespace VulnUserCodeAnalyzer ...@@ -133,86 +216,138 @@ namespace VulnUserCodeAnalyzer
133 } 216 }
134 } 217 }
135 } 218 }
136 - 219 + var findCveDict = new Dictionary<string, List<VulnAbstractCrawler.UserBlock>>();
220 + var findCveList = new HashSet<string>();
221 + /* 본격적인 취약점 매칭 부분 */
137 foreach (var vulnSet in vulnDict) 222 foreach (var vulnSet in vulnDict)
138 { 223 {
139 - Console.WriteLine($"-----cve:{vulnSet.Key}"); 224 + //Console.WriteLine($"-----cve:{vulnSet.Key}");
140 bool match = false; 225 bool match = false;
141 foreach (var vuln in vulnSet.Value) 226 foreach (var vuln in vulnSet.Value)
142 { 227 {
143 - 228 + /* 사용자 코드 해쉬 저장해논 bloom filter에 취약점 레코드 해쉬값들이 포함되는지 확인
229 + * 포함이 된다는 건 해당 취약점 레코드가 사용자 코드에도 있다는 뜻(취약점)
230 + * 같은 종류의 CVE 레코드가 전부 필터에 포함된다면 취약점으로 판단한다.
231 + */
144 if (filter.Contains(vuln.BlockHash)) 232 if (filter.Contains(vuln.BlockHash))
145 { 233 {
146 - Console.WriteLine($"필터 확인 : {vuln.BlockHash}");
147 if (hashDict.ContainsKey(vuln.LenFunc)) 234 if (hashDict.ContainsKey(vuln.LenFunc))
148 { 235 {
236 + /* Bloom Filter는 아쉽게도 포함 여부만 알 수 있기에
237 + * 포함되었음을 알았다면 검색해서 정보를 구한다. */
149 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash); 238 var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
150 if (userBlock == null) 239 if (userBlock == null)
151 { 240 {
152 - Console.WriteLine("userBlock이 비어있습니다.");
153 continue; 241 continue;
154 } 242 }
155 - Console.WriteLine($"CVE:{vuln.Cve}, {userBlock.FuncName}, 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}"); 243 + /* 해당 유저 블록을 임시 저장한다.
244 + * 밑에서 블록 정보를 DB로 전송하기 위해서다.
245 + */
246 + if (!findCveDict.ContainsKey(vuln.Cve))
247 + {
248 + findCveDict[vuln.Cve] = new List<VulnAbstractCrawler.UserBlock>();
249 + }
250 + userBlock.Url = vuln.Url;
251 + findCveDict[vuln.Cve].Add(userBlock);
156 match = true; 252 match = true;
157 - findBlocks.Enqueue(userBlock);
158 } 253 }
159 } 254 }
160 else 255 else
161 { 256 {
162 match = false; 257 match = false;
163 - // break; 258 + break;
164 } 259 }
165 } 260 }
261 + /* 취약점 레코드가 전부 있어야 CVE 찾음 인정 */
166 if (match) 262 if (match)
167 { 263 {
168 Console.WriteLine($"CVE 찾음 {vulnSet.Key}"); 264 Console.WriteLine($"CVE 찾음 {vulnSet.Key}");
265 + /* 찾았으면 cve값을 기록함 밑에서 찾은 cve 정보 전송하기 위해 */
266 + findCveList.Add(vulnSet.Key);
169 } 267 }
170 else 268 else
171 { 269 {
172 Console.WriteLine("없음"); 270 Console.WriteLine("없음");
173 } 271 }
174 } 272 }
175 -
176 stopwatch.Stop(); 273 stopwatch.Stop();
177 - 274 + /* 매칭 끝 후처리 (출력, DB 전송 등) */
178 -
179 -
180 -
181 var hours = stopwatch.Elapsed.Hours; 275 var hours = stopwatch.Elapsed.Hours;
182 var minutes = stopwatch.Elapsed.Minutes; 276 var minutes = stopwatch.Elapsed.Minutes;
183 var seconds = stopwatch.Elapsed.Seconds; 277 var seconds = stopwatch.Elapsed.Seconds;
184 -
185 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}"); 278 Console.WriteLine($"경과 시간 {hours.ToString("00")}:{minutes.ToString("00")}:{seconds.ToString("00")}");
186 - 279 + Console.WriteLine($"찾은 CVE 개수 : {findCveList.Count}");
187 - 280 + var yearMatch = new Regex(@"CVE-(\d{4})-(\d+)");
188 - // CVE JSON 검색 281 + foreach (var cve in findCveList)
189 -
190 - foreach (var vuln in findBlocks)
191 { 282 {
192 - 283 + Console.WriteLine(cve);
284 + var c = yearMatch.Match(cve);
285 + int year = int.Parse(c.Groups[1].Value);
286 + if (!CVE_JSON.CveDict.ContainsKey(year))
287 + {
288 + continue;
193 } 289 }
290 + if (!CVE_JSON.CveDict[year].ContainsKey(cve))
291 + {
292 + continue;
293 + }
294 + var data = CVE_JSON.CveDict[year][cve];
194 295
195 - // 블룸 필터 테스트 296 + /* 취약점 타입 분류 */
196 - //while(true) 297 + string type = "NORMAL";
197 - //{ 298 + if (data.Detail.IndexOf("overflow", StringComparison.CurrentCultureIgnoreCase) > 0)
198 - // string key = Console.ReadLine(); 299 + {
199 - // if (key == "-1") 300 + type = "OVERFLOW";
200 - // { 301 + }
201 - // break; 302 + else if (data.Detail.IndexOf("xss", StringComparison.CurrentCultureIgnoreCase) > 0)
202 - // } 303 + {
203 - // if (filter.Contains(key)) 304 + type = "XSS";
204 - // { 305 + }
205 - // Console.WriteLine("포함"); 306 + else if (data.Detail.IndexOf("injection", StringComparison.CurrentCultureIgnoreCase) > 0)
206 - // } 307 + {
207 - // else 308 + type = "SQLINJECTION";
208 - // { 309 + }
209 - // Console.WriteLine("없음"); 310 + else if (data.Detail.IndexOf("dos", StringComparison.CurrentCultureIgnoreCase) > 0)
210 - // } 311 + {
211 - 312 + type = "DOS";
212 - 313 + }
213 - //} 314 + else if (data.Detail.IndexOf("Memory", StringComparison.CurrentCultureIgnoreCase) > 0)
214 - 315 + {
316 + type = "MEMORY";
317 + }
318 + else if (data.Detail.IndexOf("CSRF", StringComparison.CurrentCultureIgnoreCase) > 0)
319 + {
320 + type = "CSRF";
321 + }
322 + else if (data.Detail.IndexOf("inclusion", StringComparison.CurrentCultureIgnoreCase) > 0)
323 + {
324 + type = "FILEINCLUSION";
325 + }
326 + else if (data.Detail.IndexOf("EXCUTE", StringComparison.CurrentCultureIgnoreCase) > 0)
327 + {
328 + type = "EXCUTE";
329 + }
330 + var urlBytes = Convert.FromBase64String(findCveDict[cve].FirstOrDefault().Url);
331 + string url = Encoding.Unicode.GetString(urlBytes);
215 332
333 + /* DB 전송 */
334 + VulnRDS.InsertVulnDetail(new VulnRDS.Vuln_detail
335 + {
336 + CveName = data.Code,
337 + Type = type,
338 + Level = data.Level.ToString(),
339 + Year = data.Year.ToString(),
340 + CveDetail = data.Detail,
341 + Publish_date = data.Publish_Date.ToString("yyyy-MM-dd"),
342 + Update_date = data.Update_Date.ToString("yyyy-MM-dd"),
343 + UserName = "samsung",
344 + Url = url,
345 + FileName = findCveDict[cve].FirstOrDefault().Path.Replace(@"C:\code", ""),
346 + FuncName = findCveDict[cve].FirstOrDefault().FuncName,
347 + Product = data.Type,
348 + });
349 + Console.WriteLine("추가 완료");
350 + }
216 } 351 }
217 } 352 }
218 353
......