노현종 / 2018-1-Capstone1-VulnNotti

Go to a project
Toggle navigation Toggle navigation pinning
노현종
c6c1c609 1 parent 1ef87191

1차 완성(테스트 완료)

Inline Side-by-side
Showing 8 changed files with 498 additions and 87 deletions
...@@ -21,10 +21,7 @@ namespace VulnCrawler ...@@ -21,10 +21,7 @@ namespace VulnCrawler
21 #region MySql 연결 21 #region MySql 연결
22 //SecureString s_key = GetConsoleSecurePassword(); 22 //SecureString s_key = GetConsoleSecurePassword();
23 //Console.Clear(); 23 //Console.Clear();
24 - //string key = SecureStringToString(s_key); 24 + //string key = SecureStringToString(s_key)
25 - //AWS.account.Id = "yhackerbv";
26 - //AWS.account.Pw = "guswhd12";
27 - //AWS.account.Endpoint = "vulndb.cby38wfppa7l.us-east-2.rds.amazonaws.com";
28 //AWS.SaveAccount(); 25 //AWS.SaveAccount();
29 //AES aes = new AES(); 26 //AES aes = new AES();
30 27
...@@ -81,7 +78,7 @@ namespace VulnCrawler ...@@ -81,7 +78,7 @@ namespace VulnCrawler
81 /* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */ 78 /* 폴더 중에 linux가 있으면 잠깐 넘어감 (너무 커서 테스트 힘듦) */
82 if (directory.Contains("linux")) 79 if (directory.Contains("linux"))
83 { 80 {
84 - continue; 81 + // continue;
85 } 82 }
86 // 템플릿 패턴화 T : VulnAbstractCrawler 83 // 템플릿 패턴화 T : VulnAbstractCrawler
87 VulnWorker.Run<VulnC>(directory); 84 VulnWorker.Run<VulnC>(directory);
......
...@@ -65,7 +65,11 @@ namespace VulnCrawler ...@@ -65,7 +65,11 @@ namespace VulnCrawler
65 } 65 }
66 // 소멸자 66 // 소멸자
67 ~VulnAbstractCrawler() { 67 ~VulnAbstractCrawler() {
68 - Repository?.Dispose(); 68 + try
69 + {
70 + Repository?.Dispose();
71 + }
72 + catch { }
69 } 73 }
70 74
71 private void LoadReservedList() 75 private void LoadReservedList()
...@@ -223,7 +227,7 @@ namespace VulnCrawler ...@@ -223,7 +227,7 @@ namespace VulnCrawler
223 // 괄호가 모두 닫혔으니 종료 227 // 괄호가 모두 닫혔으니 종료
224 if (bracketCount < 0) 228 if (bracketCount < 0)
225 { 229 {
226 - Console.WriteLine("괄호끝"); 230 + // Console.WriteLine("괄호끝");
227 break; 231 break;
228 } 232 }
229 // oldBuilder.AppendLine(line); 233 // oldBuilder.AppendLine(line);
...@@ -271,34 +275,36 @@ namespace VulnCrawler ...@@ -271,34 +275,36 @@ namespace VulnCrawler
271 string md5 = string.Empty; 275 string md5 = string.Empty;
272 if (item.Value.Count() != 0) 276 if (item.Value.Count() != 0)
273 { 277 {
274 - Console.WriteLine("크리티컬 변수 목록"); 278 + //Console.WriteLine("크리티컬 변수 목록");
275 - Console.ForegroundColor = ConsoleColor.Cyan; 279 + //Console.ForegroundColor = ConsoleColor.Cyan;
276 - foreach (var c in item.Value) 280 + //foreach (var c in item.Value)
277 - { 281 + //{
278 - Console.WriteLine(c); 282 + // Console.WriteLine(c);
279 - } 283 + //}
280 - Console.ResetColor(); 284 + //Console.ResetColor();
281 - Console.WriteLine("-------------------"); 285 + //Console.WriteLine("-------------------");
282 // 크리티컬 블록 추출 286 // 크리티컬 블록 추출
283 - var blocks = GetCriticalBlocks(func, item.Value).ToList(); 287 + var blocks = new List<Block>();
284 - if (blocks == null) 288 + //var blocks = GetCriticalBlocks(func, item.Value).ToList();
285 - { 289 + //if (blocks == null)
286 - continue; 290 + //{
287 - } 291 + // continue;
288 - foreach (var block in blocks) 292 + //}
289 - { 293 + //foreach (var block in blocks)
294 + //{
290 295
291 - block.CriticalList = item.Value; 296 + // block.CriticalList = item.Value;
292 - /* 추상화 및 정규화 */ 297 + // /* 추상화 및 정규화 */
293 - block.AbsCode = Abstract(block.Code, varTable, methodTable); 298 + // block.AbsCode = Abstract(block.Code, varTable, methodTable);
294 - block.Hash = MD5HashFunc(block.AbsCode); 299 + // block.Hash = MD5HashFunc(block.AbsCode);
295 300
296 - } 301 + //}
297 /* 추상화 변환 테이블 출력 */ 302 /* 추상화 변환 테이블 출력 */
298 - foreach (var var in varTable) 303 + //foreach (var var in varTable)
299 - { 304 + //{
300 - Console.WriteLine($"{var.Key}, {var.Value}"); 305 + // Console.WriteLine($"{var.Key}, {var.Value}");
301 - } 306 + //}
307 +
302 yield return (methodName, func, blocks); 308 yield return (methodName, func, blocks);
303 } 309 }
304 310
...@@ -644,7 +650,7 @@ namespace VulnCrawler ...@@ -644,7 +650,7 @@ namespace VulnCrawler
644 /// </summary> 650 /// </summary>
645 /// <param name="str">INPUT 문자열</param> 651 /// <param name="str">INPUT 문자열</param>
646 /// <returns>결과 문자열</returns> 652 /// <returns>결과 문자열</returns>
647 - protected static string MD5HashFunc(string str) { 653 + public static string MD5HashFunc(string str) {
648 StringBuilder MD5Str = new StringBuilder(); 654 StringBuilder MD5Str = new StringBuilder();
649 byte[] byteArr = Encoding.ASCII.GetBytes(str); 655 byte[] byteArr = Encoding.ASCII.GetBytes(str);
650 byte[] resultArr = (new MD5CryptoServiceProvider()).ComputeHash(byteArr); 656 byte[] resultArr = (new MD5CryptoServiceProvider()).ComputeHash(byteArr);
......
...@@ -26,7 +26,8 @@ namespace VulnCrawler ...@@ -26,7 +26,8 @@ namespace VulnCrawler
26 /// <param name="patchCode">패치 코드</param> 26 /// <param name="patchCode">패치 코드</param>
27 /// <returns></returns> 27 /// <returns></returns>
28 public override MatchCollection GetMatches(string patchCode) { 28 public override MatchCollection GetMatches(string patchCode) {
29 - var regs = Regex.Matches(patchCode, RegexFuncPattern); 29 + var funcPattern = $@"(?<{MethodName}>(unsigned|static)?( const )? [\w]+ [\w]+\(([\w \*\,\t\n])*[\)\,])";
30 + var regs = Regex.Matches(patchCode, funcPattern);
30 return regs; 31 return regs;
31 } 32 }
32 /// <summary> 33 /// <summary>
...@@ -82,7 +83,7 @@ namespace VulnCrawler ...@@ -82,7 +83,7 @@ namespace VulnCrawler
82 { 83 {
83 continue; 84 continue;
84 } 85 }
85 - Console.WriteLine(line); 86 + // Console.WriteLine(line);
86 builder.AppendLine(line); 87 builder.AppendLine(line);
87 continue; 88 continue;
88 } 89 }
......
...@@ -139,7 +139,9 @@ namespace VulnCrawler ...@@ -139,7 +139,9 @@ namespace VulnCrawler
139 //콘솔출력용 139 //콘솔출력용
140 sql = "INSERT INTO vuln_Info(vulnId, cve, funcName, lenFunc, code, blockHash, url) " + 140 sql = "INSERT INTO vuln_Info(vulnId, cve, funcName, lenFunc, code, blockHash, url) " +
141 $"VALUES({last_vulnId}, {vuln.Cve}, '{vuln.FuncName}', '{vuln.LenFunc}', {vuln.Code},'{vuln.BlockHash}', '{vuln.Url}')"; 141 $"VALUES({last_vulnId}, {vuln.Cve}, '{vuln.FuncName}', '{vuln.LenFunc}', {vuln.Code},'{vuln.BlockHash}', '{vuln.Url}')";
142 - Console.WriteLine(sql); 142 + //Console.WriteLine(sql);
143 + //Console.ReadLine();
144 +
143 } 145 }
144 catch (Exception e) 146 catch (Exception e)
145 { 147 {
...@@ -150,7 +152,7 @@ namespace VulnCrawler ...@@ -150,7 +152,7 @@ namespace VulnCrawler
150 Connect(Account, DbName); 152 Connect(Account, DbName);
151 goto Retry; 153 goto Retry;
152 } 154 }
153 - Console.ReadLine(); 155 + //Console.ReadLine();
154 } 156 }
155 } 157 }
156 public static void InsertUserData(User user) 158 public static void InsertUserData(User user)
......
...@@ -20,26 +20,40 @@ namespace VulnCrawler ...@@ -20,26 +20,40 @@ namespace VulnCrawler
20 crawler.Init(dirPath); 20 crawler.Init(dirPath);
21 /* 초기화된 커밋 목록 가져옴 */ 21 /* 초기화된 커밋 목록 가져옴 */
22 var commits = crawler.Commits; 22 var commits = crawler.Commits;
23 + int totalCount = commits.Count();
24 + int count = 0;
23 foreach (var commit in commits) { 25 foreach (var commit in commits) {
24 // 커밋 메시지 26 // 커밋 메시지
27 + count++;
28 + double per = ((double)count / (double)totalCount) * 100;
29 +
30 + Console.Clear();
31 + Console.WriteLine($"{count} / {totalCount} :: {per.ToString("#0.0")}%");
32 +
25 string message = commit.Message; 33 string message = commit.Message;
26 string cve = crawler.GetCVE(message); 34 string cve = crawler.GetCVE(message);
27 if (string.IsNullOrEmpty(cve)) { 35 if (string.IsNullOrEmpty(cve)) {
28 continue; 36 continue;
29 } 37 }
38 +
30 foreach (var parent in commit.Parents) { 39 foreach (var parent in commit.Parents) {
31 40
32 - // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴 41 + try
33 - var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree); 42 + {
34 - // 패치 엔트리 파일 배열 중에 파일 확장자가 .py인 것만 가져옴 43 + // 부모 커밋과 현재 커밋을 Compare 하여 패치 내역을 가져옴
35 - // (실질적인 코드 변경 커밋만 보기 위해서) 44 + var patch = crawler.Repository.Diff.Compare<Patch>(parent.Tree, commit.Tree);
36 - var entrys = crawler.GetPatchEntryChanges(patch); 45 + // 패치 엔트리 파일 배열 중에 파일 확장자가 .py인 것만 가져옴
37 - /* C:\VulnC\linux 라면 linux만 뽑아서 repoName에 저장 */ 46 + // (실질적인 코드 변경 커밋만 보기 위해서)
38 - var dsp = dirPath.Split(Path.DirectorySeparatorChar); 47 + var entrys = crawler.GetPatchEntryChanges(patch);
39 - string repoName = dsp[dsp.Length - 1]; 48 + /* C:\VulnC\linux 라면 linux만 뽑아서 repoName에 저장 */
40 - // 현재 커밋에 대한 패치 엔트리 배열을 출력함 49 + var dsp = dirPath.Split(Path.DirectorySeparatorChar);
41 - PrintPatchEntrys(entrys, crawler, message, cve, repoName); 50 + string repoName = dsp[dsp.Length - 1];
42 - // Console.ReadLine(); 51 + // 현재 커밋에 대한 패치 엔트리 배열을 출력함
52 + PrintPatchEntrys(entrys, crawler, message, cve, repoName);
53 + // Console.ReadLine();
54 + }
55 + catch(Exception)
56 + { }
43 } 57 }
44 } 58 }
45 } 59 }
...@@ -64,33 +78,33 @@ namespace VulnCrawler ...@@ -64,33 +78,33 @@ namespace VulnCrawler
64 // 출력 78 // 출력
65 if (regs.Count > 0) 79 if (regs.Count > 0)
66 { 80 {
67 - int deleted = entry.LinesDeleted; 81 + //int deleted = entry.LinesDeleted;
68 - if (deleted == 0) 82 + //if (deleted == 0)
69 - { 83 + //{
70 - continue; 84 + // // continue;
71 - } 85 + //}
72 - Console.BackgroundColor = ConsoleColor.DarkBlue; 86 + //Console.BackgroundColor = ConsoleColor.DarkBlue;
73 - Console.WriteLine($"Old Content: \n{oldContent}"); 87 + //Console.WriteLine($"Old Content: \n{oldContent}");
74 - Console.ResetColor(); 88 + //Console.ResetColor();
75 - 89 +
76 - Console.ForegroundColor = ConsoleColor.Blue; 90 + //Console.ForegroundColor = ConsoleColor.Blue;
77 - Console.WriteLine($"status: {entry.Status.ToString()}"); 91 + //Console.WriteLine($"status: {entry.Status.ToString()}");
78 - Console.WriteLine($"added: {entry.LinesAdded.ToString()}, deleted: {entry.LinesDeleted.ToString()}"); 92 + //Console.WriteLine($"added: {entry.LinesAdded.ToString()}, deleted: {entry.LinesDeleted.ToString()}");
79 - Console.WriteLine($"old path: {entry.OldPath.ToString()}, new path: {entry.Path.ToString()}"); 93 + //Console.WriteLine($"old path: {entry.OldPath.ToString()}, new path: {entry.Path.ToString()}");
80 - Console.ResetColor(); 94 + //Console.ResetColor();
81 - 95 +
82 - 96 +
83 - Console.Write($"CVE: "); 97 + //Console.Write($"CVE: ");
84 - Console.ForegroundColor = ConsoleColor.Red; 98 + //Console.ForegroundColor = ConsoleColor.Red;
85 - Console.Write($"{cve}"); 99 + //Console.Write($"{cve}");
86 - Console.WriteLine(""); 100 + //Console.WriteLine("");
87 - Console.ResetColor(); 101 + //Console.ResetColor();
88 - Console.ForegroundColor = ConsoleColor.Yellow; 102 + //Console.ForegroundColor = ConsoleColor.Yellow;
89 - Console.WriteLine($"Commit Message: {commitMsg}"); 103 + //Console.WriteLine($"Commit Message: {commitMsg}");
90 - Console.ResetColor(); 104 + //Console.ResetColor();
91 - Console.BackgroundColor = ConsoleColor.DarkRed; 105 + //Console.BackgroundColor = ConsoleColor.DarkRed;
92 - Console.WriteLine($"Patched: \n{entry.Patch}"); 106 + //Console.WriteLine($"Patched: \n{entry.Patch}");
93 - Console.ResetColor(); 107 + //Console.ResetColor();
94 /* 패치된 코드들에서 Method로 나누고 크리티컬 변수로 뽑아옴 Dictionary 구조 (키 = 함수명) */ 108 /* 패치된 코드들에서 Method로 나누고 크리티컬 변수로 뽑아옴 Dictionary 구조 (키 = 함수명) */
95 var table = self.ExtractGitCriticalMethodTable(entry.Patch); 109 var table = self.ExtractGitCriticalMethodTable(entry.Patch);
96 /* 크리티컬 메서드 테이블과 패치 전 파일에서 Process 하고 tuple로 가져옴 */ 110 /* 크리티컬 메서드 테이블과 패치 전 파일에서 Process 하고 tuple로 가져옴 */
...@@ -98,10 +112,10 @@ namespace VulnCrawler ...@@ -98,10 +112,10 @@ namespace VulnCrawler
98 { 112 {
99 /* 메서드 이름, 원본 함수 코드, 블록 리스트(크리티컬 포함) */ 113 /* 메서드 이름, 원본 함수 코드, 블록 리스트(크리티컬 포함) */
100 (var methodName, var oriFunc, var blocks) = tuple; 114 (var methodName, var oriFunc, var blocks) = tuple;
101 - Console.BackgroundColor = ConsoleColor.DarkRed; 115 + //Console.BackgroundColor = ConsoleColor.DarkRed;
102 - Console.WriteLine($"메서드 이름 : {methodName}"); 116 + //Console.WriteLine($"메서드 이름 : {methodName}");
103 - Console.ResetColor(); 117 + //Console.ResetColor();
104 - //foreach (var block in blocks) 118 + ////foreach (var block in blocks)
105 //{ 119 //{
106 // /* 크리티컬 블록이 아니면 볼 필요 없으니 넘어감 */ 120 // /* 크리티컬 블록이 아니면 볼 필요 없으니 넘어감 */
107 // if (!block.HasCritical) 121 // if (!block.HasCritical)
...@@ -146,26 +160,41 @@ namespace VulnCrawler ...@@ -146,26 +160,41 @@ namespace VulnCrawler
146 /* VulnDB에 추가 */ 160 /* VulnDB에 추가 */
147 //VulnRDS.InsertVulnData(vuln); 161 //VulnRDS.InsertVulnData(vuln);
148 //} 162 //}
163 +
164 + if (string.IsNullOrWhiteSpace(oriFunc))
165 + {
166 + continue;
167 + }
149 string abstractCode = self.Abstract(oriFunc, new Dictionary<string, string>(), new Dictionary<string, string>()); 168 string abstractCode = self.Abstract(oriFunc, new Dictionary<string, string>(), new Dictionary<string, string>());
150 169
151 byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName); 170 byte[] funcNameBytes = Encoding.Unicode.GetBytes(methodName);
152 byte[] absCodeBytes = Encoding.Unicode.GetBytes(abstractCode); 171 byte[] absCodeBytes = Encoding.Unicode.GetBytes(abstractCode);
153 - VulnRDS.Vuln vuln = new VulnRDS.Vuln() 172 + byte[] commitMsgBytes = Encoding.Unicode.GetBytes(commitMsg);
173 + byte[] funcBytes = Encoding.Unicode.GetBytes(oriFunc);
174 + VulnRDS._Vuln vuln = new VulnRDS._Vuln()
154 { 175 {
155 - BlockHash = Convert.ToBase64String(absCodeBytes), 176 + LenFunc = oriFunc.Length,
156 Cve = cve, 177 Cve = cve,
157 - LenBlock = oriFunc.Length, 178 + BlockHash = VulnAbstractCrawler.MD5HashFunc(Convert.ToBase64String(absCodeBytes)),
158 FuncName = Convert.ToBase64String(funcNameBytes), 179 FuncName = Convert.ToBase64String(funcNameBytes),
180 + Code = Convert.ToBase64String(funcBytes),
181 + Url = Convert.ToBase64String(commitMsgBytes),
182 + //BlockHash = Convert.ToBase64String(absCodeBytes),
183 + //Cve = cve,
184 + //LenBlock = oriFunc.Length,
185 + //FuncName = Convert.ToBase64String(funcNameBytes),
159 }; 186 };
160 - Console.WriteLine(vuln.BlockHash); 187 + // Console.WriteLine(vuln.BlockHash);
161 - Console.ReadLine(); 188 + // Console.ReadLine();
162 /* VulnDB에 추가 */ 189 /* VulnDB에 추가 */
163 - //VulnRDS.InsertVulnData(vuln); 190 + VulnRDS._InsertVulnData(vuln);
164 191
165 } 192 }
166 } 193 }
167 else 194 else
168 { 195 {
196 + //Console.WriteLine("zzz");
197 + //Console.ReadLine();
169 continue; 198 continue;
170 } 199 }
171 200
...@@ -174,6 +203,8 @@ namespace VulnCrawler ...@@ -174,6 +203,8 @@ namespace VulnCrawler
174 } 203 }
175 catch (Exception e) 204 catch (Exception e)
176 { 205 {
206 + //Console.WriteLine(e.ToString());
207 + //Console.ReadLine();
177 continue; 208 continue;
178 } 209 }
179 210
......
1 +using System;
2 +using System.Collections.Generic;
3 +using System.Linq;
4 +using System.Text;
5 +using System.Threading.Tasks;
6 +
7 +namespace BloomFilter
8 +{
9 + using System;
10 + using System.Collections;
11 +
12 + /// <summary>
13 + /// Bloom filter.
14 + /// </summary>
15 + /// <typeparam name="T">Item type </typeparam>
16 + public class Filter<T>
17 + {
18 + private readonly int _hashFunctionCount;
19 + private readonly BitArray _hashBits;
20 + private readonly HashFunction _getHashSecondary;
21 +
22 + /// <summary>
23 + /// Creates a new Bloom filter, specifying an error rate of 1/capacity, using the optimal size for the underlying data structure based on the desired capacity and error rate, as well as the optimal number of hash functions.
24 + /// A secondary hash function will be provided for you if your type T is either string or int. Otherwise an exception will be thrown. If you are not using these types please use the overload that supports custom hash functions.
25 + /// </summary>
26 + /// <param name="capacity">The anticipated number of items to be added to the filter. More than this number of items can be added, but the error rate will exceed what is expected.</param>
27 + public Filter(int capacity)
28 + : this(capacity, null)
29 + {
30 + }
31 +
32 + /// <summary>
33 + /// Creates a new Bloom filter, using the optimal size for the underlying data structure based on the desired capacity and error rate, as well as the optimal number of hash functions.
34 + /// A secondary hash function will be provided for you if your type T is either string or int. Otherwise an exception will be thrown. If you are not using these types please use the overload that supports custom hash functions.
35 + /// </summary>
36 + /// <param name="capacity">The anticipated number of items to be added to the filter. More than this number of items can be added, but the error rate will exceed what is expected.</param>
37 + /// <param name="errorRate">The accepable false-positive rate (e.g., 0.01F = 1%)</param>
38 + public Filter(int capacity, float errorRate)
39 + : this(capacity, errorRate, null)
40 + {
41 + }
42 +
43 + /// <summary>
44 + /// Creates a new Bloom filter, specifying an error rate of 1/capacity, using the optimal size for the underlying data structure based on the desired capacity and error rate, as well as the optimal number of hash functions.
45 + /// </summary>
46 + /// <param name="capacity">The anticipated number of items to be added to the filter. More than this number of items can be added, but the error rate will exceed what is expected.</param>
47 + /// <param name="hashFunction">The function to hash the input values. Do not use GetHashCode(). If it is null, and T is string or int a hash function will be provided for you.</param>
48 + public Filter(int capacity, HashFunction hashFunction)
49 + : this(capacity, BestErrorRate(capacity), hashFunction)
50 + {
51 + }
52 +
53 + /// <summary>
54 + /// Creates a new Bloom filter, using the optimal size for the underlying data structure based on the desired capacity and error rate, as well as the optimal number of hash functions.
55 + /// </summary>
56 + /// <param name="capacity">The anticipated number of items to be added to the filter. More than this number of items can be added, but the error rate will exceed what is expected.</param>
57 + /// <param name="errorRate">The accepable false-positive rate (e.g., 0.01F = 1%)</param>
58 + /// <param name="hashFunction">The function to hash the input values. Do not use GetHashCode(). If it is null, and T is string or int a hash function will be provided for you.</param>
59 + public Filter(int capacity, float errorRate, HashFunction hashFunction)
60 + : this(capacity, errorRate, hashFunction, BestM(capacity, errorRate), BestK(capacity, errorRate))
61 + {
62 + }
63 +
64 + /// <summary>
65 + /// Creates a new Bloom filter.
66 + /// </summary>
67 + /// <param name="capacity">The anticipated number of items to be added to the filter. More than this number of items can be added, but the error rate will exceed what is expected.</param>
68 + /// <param name="errorRate">The accepable false-positive rate (e.g., 0.01F = 1%)</param>
69 + /// <param name="hashFunction">The function to hash the input values. Do not use GetHashCode(). If it is null, and T is string or int a hash function will be provided for you.</param>
70 + /// <param name="m">The number of elements in the BitArray.</param>
71 + /// <param name="k">The number of hash functions to use.</param>
72 + public Filter(int capacity, float errorRate, HashFunction hashFunction, int m, int k)
73 + {
74 + // validate the params are in range
75 + if (capacity < 1)
76 + {
77 + throw new ArgumentOutOfRangeException("capacity", capacity, "capacity must be > 0");
78 + }
79 +
80 + if (errorRate >= 1 || errorRate <= 0)
81 + {
82 + throw new ArgumentOutOfRangeException("errorRate", errorRate, string.Format("errorRate must be between 0 and 1, exclusive. Was {0}", errorRate));
83 + }
84 +
85 + // from overflow in bestM calculation
86 + if (m < 1)
87 + {
88 + throw new ArgumentOutOfRangeException(string.Format("The provided capacity and errorRate values would result in an array of length > int.MaxValue. Please reduce either of these values. Capacity: {0}, Error rate: {1}", capacity, errorRate));
89 + }
90 +
91 + // set the secondary hash function
92 + if (hashFunction == null)
93 + {
94 + if (typeof(T) == typeof(string))
95 + {
96 + this._getHashSecondary = HashString;
97 + }
98 + else if (typeof(T) == typeof(int))
99 + {
100 + this._getHashSecondary = HashInt32;
101 + }
102 + else
103 + {
104 + throw new ArgumentNullException("hashFunction", "Please provide a hash function for your type T, when T is not a string or int.");
105 + }
106 + }
107 + else
108 + {
109 + this._getHashSecondary = hashFunction;
110 + }
111 +
112 + this._hashFunctionCount = k;
113 + this._hashBits = new BitArray(m);
114 + }
115 +
116 + /// <summary>
117 + /// A function that can be used to hash input.
118 + /// </summary>
119 + /// <param name="input">The values to be hashed.</param>
120 + /// <returns>The resulting hash code.</returns>
121 + public delegate int HashFunction(T input);
122 +
123 + /// <summary>
124 + /// The ratio of false to true bits in the filter. E.g., 1 true bit in a 10 bit filter means a truthiness of 0.1.
125 + /// </summary>
126 + public double Truthiness {
127 + get {
128 + return (double)this.TrueBits() / this._hashBits.Count;
129 + }
130 + }
131 +
132 + /// <summary>
133 + /// Adds a new item to the filter. It cannot be removed.
134 + /// </summary>
135 + /// <param name="item">The item.</param>
136 + public void Add(T item)
137 + {
138 + // start flipping bits for each hash of item
139 + int primaryHash = item.GetHashCode();
140 + int secondaryHash = this._getHashSecondary(item);
141 + for (int i = 0; i < this._hashFunctionCount; i++)
142 + {
143 + int hash = this.ComputeHash(primaryHash, secondaryHash, i);
144 + this._hashBits[hash] = true;
145 + }
146 + }
147 +
148 + /// <summary>
149 + /// Checks for the existance of the item in the filter for a given probability.
150 + /// </summary>
151 + /// <param name="item"> The item. </param>
152 + /// <returns> The <see cref="bool"/>. </returns>
153 + public bool Contains(T item)
154 + {
155 + int primaryHash = item.GetHashCode();
156 + int secondaryHash = this._getHashSecondary(item);
157 + for (int i = 0; i < this._hashFunctionCount; i++)
158 + {
159 + int hash = this.ComputeHash(primaryHash, secondaryHash, i);
160 + if (this._hashBits[hash] == false)
161 + {
162 + return false;
163 + }
164 + }
165 +
166 + return true;
167 + }
168 +
169 + /// <summary>
170 + /// The best k.
171 + /// </summary>
172 + /// <param name="capacity"> The capacity. </param>
173 + /// <param name="errorRate"> The error rate. </param>
174 + /// <returns> The <see cref="int"/>. </returns>
175 + private static int BestK(int capacity, float errorRate)
176 + {
177 + return (int)Math.Round(Math.Log(2.0) * BestM(capacity, errorRate) / capacity);
178 + }
179 +
180 + /// <summary>
181 + /// The best m.
182 + /// </summary>
183 + /// <param name="capacity"> The capacity. </param>
184 + /// <param name="errorRate"> The error rate. </param>
185 + /// <returns> The <see cref="int"/>. </returns>
186 + private static int BestM(int capacity, float errorRate)
187 + {
188 + return (int)Math.Ceiling(capacity * Math.Log(errorRate, (1.0 / Math.Pow(2, Math.Log(2.0)))));
189 + }
190 +
191 + /// <summary>
192 + /// The best error rate.
193 + /// </summary>
194 + /// <param name="capacity"> The capacity. </param>
195 + /// <returns> The <see cref="float"/>. </returns>
196 + private static float BestErrorRate(int capacity)
197 + {
198 + float c = (float)(1.0 / capacity);
199 + if (c != 0)
200 + {
201 + return c;
202 + }
203 +
204 + // default
205 + // http://www.cs.princeton.edu/courses/archive/spring02/cs493/lec7.pdf
206 + return (float)Math.Pow(0.6185, int.MaxValue / capacity);
207 + }
208 +
209 + /// <summary>
210 + /// Hashes a 32-bit signed int using Thomas Wang's method v3.1 (http://www.concentric.net/~Ttwang/tech/inthash.htm).
211 + /// Runtime is suggested to be 11 cycles.
212 + /// </summary>
213 + /// <param name="input">The integer to hash.</param>
214 + /// <returns>The hashed result.</returns>
215 + private static int HashInt32(T input)
216 + {
217 + uint? x = input as uint?;
218 + unchecked
219 + {
220 + x = ~x + (x << 15); // x = (x << 15) - x- 1, as (~x) + y is equivalent to y - x - 1 in two's complement representation
221 + x = x ^ (x >> 12);
222 + x = x + (x << 2);
223 + x = x ^ (x >> 4);
224 + x = x * 2057; // x = (x + (x << 3)) + (x<< 11);
225 + x = x ^ (x >> 16);
226 + return (int)x;
227 + }
228 + }
229 +
230 + /// <summary>
231 + /// Hashes a string using Bob Jenkin's "One At A Time" method from Dr. Dobbs (http://burtleburtle.net/bob/hash/doobs.html).
232 + /// Runtime is suggested to be 9x+9, where x = input.Length.
233 + /// </summary>
234 + /// <param name="input">The string to hash.</param>
235 + /// <returns>The hashed result.</returns>
236 + private static int HashString(T input)
237 + {
238 + string s = input as string;
239 + int hash = 0;
240 +
241 + for (int i = 0; i < s.Length; i++)
242 + {
243 + hash += s[i];
244 + hash += (hash << 10);
245 + hash ^= (hash >> 6);
246 + }
247 +
248 + hash += (hash << 3);
249 + hash ^= (hash >> 11);
250 + hash += (hash << 15);
251 + return hash;
252 + }
253 +
254 + /// <summary>
255 + /// The true bits.
256 + /// </summary>
257 + /// <returns> The <see cref="int"/>. </returns>
258 + private int TrueBits()
259 + {
260 + int output = 0;
261 + foreach (bool bit in this._hashBits)
262 + {
263 + if (bit == true)
264 + {
265 + output++;
266 + }
267 + }
268 +
269 + return output;
270 + }
271 +
272 + /// <summary>
273 + /// Performs Dillinger and Manolios double hashing.
274 + /// </summary>
275 + /// <param name="primaryHash"> The primary hash. </param>
276 + /// <param name="secondaryHash"> The secondary hash. </param>
277 + /// <param name="i"> The i. </param>
278 + /// <returns> The <see cref="int"/>. </returns>
279 + private int ComputeHash(int primaryHash, int secondaryHash, int i)
280 + {
281 + int resultingHash = (primaryHash + (i * secondaryHash)) % this._hashBits.Count;
282 + return Math.Abs((int)resultingHash);
283 + }
284 + }
285 +}
1  1 
2 +using BloomFilter;
2 using System; 3 using System;
3 using System.Collections.Generic; 4 using System.Collections.Generic;
4 using System.IO; 5 using System.IO;
...@@ -14,6 +15,47 @@ namespace VulnUserCodeAnalyzer ...@@ -14,6 +15,47 @@ namespace VulnUserCodeAnalyzer
14 { 15 {
15 static void Main(string[] args) 16 static void Main(string[] args)
16 { 17 {
18 +
19 + // default usage
20 + int capacity = 20000000;
21 + var filter = new Filter<string>(capacity);
22 + //filter.Add("1");
23 + // filter.Add("1");
24 + //Console.WriteLine(filter.Contains("1"));
25 + //Console.WriteLine(filter.Contains("content2"));
26 +
27 + /* AWS 계정 정보 파일 읽음 */
28 + string txt = File.ReadAllText(@"Account.xml");
29 + // string xml = aes.AESDecrypt128(txt, key);
30 + string xml = txt;
31 +
32 + AWS.LoadAccount(xml);
33 + AWS.Account account = AWS.account;
34 +
35 + /* AWS 정보 출력 */
36 + Console.WriteLine($"Endpoint: {account.Endpoint}, ID: {account.Id}, PW: {account.Pw}");
37 + try
38 + {
39 + /* DB 접속 시도 */
40 + VulnRDS.Connect(account, "vuln");
41 + }
42 + catch (Exception e)
43 + {
44 + Console.WriteLine($"접속 에러 :: {e.ToString()}");
45 + }
46 +
47 + /* AWS 연결 여부 확인 */
48 + if (VulnRDS.Conn.State == System.Data.ConnectionState.Open)
49 + {
50 + Console.WriteLine("접속 성공");
51 +
52 + }
53 + else
54 + {
55 + Console.WriteLine("연결 실패");
56 + return;
57 + }
58 +
17 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>(); 59 var hashDict = new Dictionary<int, HashSet<VulnAbstractCrawler.UserBlock>>();
18 60
19 DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code"); 61 DirectoryInfo dirInfo = new DirectoryInfo(@"c:\code");
...@@ -39,6 +81,7 @@ namespace VulnUserCodeAnalyzer ...@@ -39,6 +81,7 @@ namespace VulnUserCodeAnalyzer
39 { 81 {
40 hash.Path = codeFile.FullName; 82 hash.Path = codeFile.FullName;
41 hashDict[item.Key].Add(hash); 83 hashDict[item.Key].Add(hash);
84 + filter.Add(hash.Hash);
42 } 85 }
43 } 86 }
44 87
...@@ -48,7 +91,7 @@ namespace VulnUserCodeAnalyzer ...@@ -48,7 +91,7 @@ namespace VulnUserCodeAnalyzer
48 Console.Clear(); 91 Console.Clear();
49 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}"); 92 Console.WriteLine($"{count} / {totalFileCount} :: {per.ToString("#0.0")}%, 개체 수 : {hashDict.Count}");
50 93
51 - //if (count > 20) 94 + //if (count > 100)
52 //{ 95 //{
53 // break; 96 // break;
54 //} 97 //}
...@@ -60,13 +103,57 @@ namespace VulnUserCodeAnalyzer ...@@ -60,13 +103,57 @@ namespace VulnUserCodeAnalyzer
60 foreach (var set in hashDict) 103 foreach (var set in hashDict)
61 { 104 {
62 Console.WriteLine($"-----key:{set.Key}"); 105 Console.WriteLine($"-----key:{set.Key}");
63 - foreach (var hash in set.Value) 106 + var vulnList = VulnRDS.SelectVulnbyLen(set.Key);
107 + foreach (var vuln in vulnList)
64 { 108 {
65 - Console.WriteLine($"{hash.FuncName}, {hash.Hash}, {hash.Len}, {hash.Path}"); 109 + // Console.WriteLine(vuln.BlockHash);
110 + if (filter.Contains(vuln.BlockHash))
111 + {
112 + Console.WriteLine($"필터 확인 : {vuln.BlockHash}");
113 + if (hashDict.ContainsKey(vuln.LenFunc))
114 + {
115 + var userBlock = hashDict[vuln.LenFunc].FirstOrDefault(b => b.Hash == vuln.BlockHash);
116 + if (userBlock == null)
117 + {
118 + Console.WriteLine("userBlock이 비어있습니다.");
119 + continue;
120 + }
121 +
122 + Console.WriteLine($"{userBlock.FuncName} 블록 확인 : DB : {vuln.BlockHash}, User : {userBlock.Hash}");
123 +
124 +
125 + }
126 + }
127 +
66 } 128 }
129 + //foreach (var hash in set.Value)
130 + //{
131 +
132 + // Console.WriteLine($"{hash.FuncName}, {hash.Hash}, {hash.Len}, {hash.Path}");
133 + //}
67 } 134 }
68 135
69 136
137 + // 블룸 필터 테스트
138 + //while(true)
139 + //{
140 + // string key = Console.ReadLine();
141 + // if (key == "-1")
142 + // {
143 + // break;
144 + // }
145 + // if (filter.Contains(key))
146 + // {
147 + // Console.WriteLine("포함");
148 + // }
149 + // else
150 + // {
151 + // Console.WriteLine("없음");
152 + // }
153 +
154 +
155 + //}
156 +
70 157
71 } 158 }
72 } 159 }
......
...@@ -32,6 +32,7 @@ ...@@ -32,6 +32,7 @@
32 <WarningLevel>4</WarningLevel> 32 <WarningLevel>4</WarningLevel>
33 </PropertyGroup> 33 </PropertyGroup>
34 <ItemGroup> 34 <ItemGroup>
35 + <Reference Include="MySql.Data, Version=8.0.10.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL" />
35 <Reference Include="System" /> 36 <Reference Include="System" />
36 <Reference Include="System.Core" /> 37 <Reference Include="System.Core" />
37 <Reference Include="System.Xml.Linq" /> 38 <Reference Include="System.Xml.Linq" />
...@@ -42,6 +43,7 @@ ...@@ -42,6 +43,7 @@
42 <Reference Include="System.Xml" /> 43 <Reference Include="System.Xml" />
43 </ItemGroup> 44 </ItemGroup>
44 <ItemGroup> 45 <ItemGroup>
46 + <Compile Include="BloomFilter.cs" />
45 <Compile Include="Program.cs" /> 47 <Compile Include="Program.cs" />
46 <Compile Include="Properties\AssemblyInfo.cs" /> 48 <Compile Include="Properties\AssemblyInfo.cs" />
47 </ItemGroup> 49 </ItemGroup>
......