Repository navigation
Expand file tree
/
Copy pathbuild_exploit_index.py
More file actions
338 lines (287 loc) · 14.5 KB
/
Copy pathbuild_exploit_index.py
File metadata and controls
338 lines (287 loc) · 14.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
#!/usr/bin/env python3
"""
Build SQLite index of exploit repositories.
Scans repositories for CVE references, platforms, exploit types.
"""
import os
import re
import sqlite3
import json
import sys
from pathlib import Path
from datetime import datetime
# Repository paths to scan
REPO_PATHS = {
"0xdea-exploits": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\0xdea-exploits",
"cve": r"C:\Projects\Quantum CLI\cve",
"exploit-db": r"C:\Projects\Quantum CLI\exploit-db",
"metasploit-framework": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\metasploit-framework",
"nuclei-templates": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\nuclei-templates",
"payloads": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\payloads",
"PayloadsAllTheThings": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\PayloadsAllTheThings",
"SharpCollection": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\SharpCollection",
"Active-Directory-Exploitation-Cheat-Sheet": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\Active-Directory-Exploitation-Cheat-Sheet",
"IntruderPayloads": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\IntruderPayloads",
"Red-Teaming-Toolkit": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\Red-Teaming-Toolkit",
"SecLists": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\SecLists",
"PoC-in-GitHub": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\PoC-in-GitHub",
"windows-kernel-exploits": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\windows-kernel-exploits",
"linux-kernel-exploits": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\linux-kernel-exploits",
"bugbounty-cheatsheet": r"C:\Projects\Quantum CLI\exploit_catalog\exploits\by_repo\bugbounty-cheatsheet",
}
DB_PATH = r"C:\Projects\Quantum CLI\exploit_catalog\exploit_db_index\top_repos.sqlite"
# CVE pattern
CVE_PATTERN = re.compile(r'CVE-\d{4}-\d{4,7}', re.IGNORECASE)
# Platform detection patterns
PLATFORM_PATTERNS = {
'windows': [r'\bwindows\b', r'\bwin32\b', r'\bwin64\b', r'\.exe\b', r'\.dll\b', r'\bps1\b', r'\bpowershell\b', r'\bcmd\b', r'\bactive.?directory\b', r'\bad\b', r'\bdomain.?controller\b'],
'linux': [r'\blinux\b', r'\bubuntu\b', r'\bdebian\b', r'\bcentos\b', r'\bredhat\b', r'\bkali\b', r'\.sh\b', r'\belf\b', r'\bso\b', r'\bbash\b', r'\bgcc\b', r'\bkernel\b'],
'macos': [r'\bmacos\b', r'\bosx\b', r'\bmac.?os\b', r'\bdarwin\b', r'\.dylib\b', r'\bmach-o\b'],
'android': [r'\bandroid\b', r'\bapk\b', r'\bdex\b', r'\baapt\b', r'\badb\b'],
'ios': [r'\bios\b', r'\biphone\b', r'\bipad\b', r'\bmacho\b', r'\.ipa\b'],
'web': [r'\bweb\b', r'\bhttp\b', r'\bhtml\b', r'\bphp\b', r'\baspx\b', r'\bjsp\b', r'\bxss\b', r'\bsqli\b', r'\bcsrf\b', r'\blfi\b', r'\brfi\b', r'\bssrf\b', r'\bapi\b'],
'network': [r'\bnetwork\b', r'\btcp\b', r'\budp\b', r'\bport\b', r'\bfirewall\b', r'\bvpn\b', r'\bssh\b', r'\bftp\b', r'\bsmb\b', r'\brdp\b'],
'database': [r'\bdatabase\b', r'\bsql\b', r'\bmysql\b', r'\bpostgres\b', r'\boracle\b', r'\bmssql\b', r'\bmongodb\b', r'\bredis\b'],
'container': [r'\bdocker\b', r'\bkubernetes\b', r'\bk8s\b', r'\bcontainer\b', r'\bpod\b'],
'cloud': [r'\baws\b', r'\bazure\b', r'\bgcp\b', r'\bcloud\b', r'\blambda\b', r'\bs3\b', r'\biam\b'],
}
# Exploit type patterns
TYPE_PATTERNS = {
'rce': [r'\brce\b', r'\bremote.?code.?execution\b', r'\bcode.?execution\b', r'\bcommand.?injection\b', r'\bshell\b', r'\breverse.?shell\b', r'\bbind.?shell\b'],
'lpe': [r'\blpe\b', r'\bprivilege.?escalation\b', r'\blocal.?privilege.?escalation\b', r'\broot\b', r'\bsystem\b', r'\badmin\b', r'\beop\b'],
'sql injection': [r'\bsql.?injection\b', r'\bsqli\b', r'\bunion.?select\b'],
'xss': [r'\bxss\b', r'\bcross.?site.?scripting\b', r'\breflected.?xss\b', r'\bstored.?xss\b', r'\bdom.?xss\b'],
'csrf': [r'\bcsrf\b', r'\bcross.?site.?request.?forgery\b'],
'ssrf': [r'\bssrf\b', r'\bserver.?side.?request.?forgery\b'],
'lfi': [r'\blfi\b', r'\blocal.?file.?inclusion\b'],
'rfi': [r'\brfi\b', r'\bremote.?file.?inclusion\b'],
'xxe': [r'\bxxe\b', r'\bxml.?external.?entity\b'],
'deserialization': [r'\bdeserialization\b', r'\bserialization\b', r'\bgadget\b', r'\bobject.?injection\b', r'\bysoserial\b'],
'buffer overflow': [r'\bbuffer.?overflow\b', r'\bstack.?overflow\b', r'\bheap.?overflow\b', r'\boverflow\b'],
'use after free': [r'\buse.?after.?free\b', r'\buaf\b'],
'race condition': [r'\brace.?condition\b', r'\btof\b', r'\btoctou\b'],
'auth bypass': [r'\bauth.?bypass\b', r'\bauthentication.?bypass\b', r'\bprivilege.?bypass\b'],
'information disclosure': [r'\binformation.?disclosure\b', r'\bleak\b', r'\bdisclosure\b', r'\benum\b', r'\brecon\b'],
'dos': [r'\bdos\b', r'\bdenial.?of.?service\b', r'\bcrash\b'],
'bypass': [r'\bbypass\b', r'\bwaf.?bypass\b', r'\bav.?bypass\b', r'\bedr.?bypass\b'],
'persistence': [r'\bpersistence\b', r'\bbackdoor\b', r'\bimplant\b', r'\brootkit\b'],
'lateral movement': [r'\blateral.?movement\b', r'\bpass.?the.?hash\b', r'\bpass.?the.?ticket\b', r'\bpth\b', r'\bptt\b'],
'credential theft': [r'\bcredential.?theft\b', r'\bcredential.?dump\b', r'\bmimikatz\b', r'\blsass\b', r'\bsekurlsa\b'],
}
def init_db():
"""Initialize the SQLite database."""
conn = sqlite3.connect(DB_PATH)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS repositories (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT UNIQUE NOT NULL,
path TEXT,
total_files INTEGER DEFAULT 0,
total_size_bytes INTEGER DEFAULT 0,
cve_count INTEGER DEFAULT 0,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER,
rel_path TEXT,
size_bytes INTEGER,
cve_refs TEXT,
platforms TEXT,
types TEXT,
FOREIGN KEY (repo_id) REFERENCES repositories(id)
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS cve_references (
id INTEGER PRIMARY KEY AUTOINCREMENT,
repo_id INTEGER,
cve_id TEXT,
file_path TEXT,
FOREIGN KEY (repo_id) REFERENCES repositories(id)
)
''')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_files_repo ON files(repo_id)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_cve_repo ON cve_references(repo_id)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_cve_id ON cve_references(cve_id)')
conn.commit()
return conn
def detect_platforms(content, filepath):
"""Detect platforms from file content and path."""
text = (content + ' ' + filepath).lower()
platforms = []
for platform, patterns in PLATFORM_PATTERNS.items():
for pattern in patterns:
if re.search(pattern, text, re.IGNORECASE):
platforms.append(platform)
break
return list(set(platforms))
def detect_types(content, filepath):
"""Detect exploit types from file content and path."""
text = (content + ' ' + filepath).lower()
types = []
for type_name, patterns in TYPE_PATTERNS.items():
for pattern in patterns:
if re.search(pattern, text, re.IGNORECASE):
types.append(type_name)
break
return list(set(types))
def extract_cves(content):
"""Extract CVE references from content."""
return list(set(CVE_PATTERN.findall(content.upper())))
def scan_file(filepath, rel_path):
"""Scan a single file for metadata."""
try:
# Skip binary files
if filepath.suffix.lower() in ['.exe', '.dll', '.so', '.dylib', '.bin', '.img', '.iso', '.zip', '.tar', '.gz', '.rar', '.7z', '.pdf', '.doc', '.docx', '.ppt', '.pptx', '.xls', '.xlsx', '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.ico', '.mp3', '.mp4', '.avi', '.mov', '.wav', '.pcap', '.cap', '.ova', '.vmdk', '.vdi', '.qcow2', '.class', '.jar', '.war', '.ear', '.apk', '.ipa', '.dmg', '.pkg', '.msi', '.deb', '.rpm', '.pyc', '.pyo', '.pyd', '.so', '.dylib']:
return None
# Skip certain directories
skip_dirs = ['node_modules', 'vendor', '__pycache__', '.git', 'dist', 'build', 'target', 'bin', 'obj', 'Debug', 'Release', 'x64', 'x86', 'packages', 'venv', 'env', '.venv', '.env']
if any(skip in rel_path.split('/') for skip in skip_dirs):
return None
size = filepath.stat().st_size
if size > 500_000: # Skip files > 500KB
return None
# Read file content
try:
content = filepath.read_text(encoding='utf-8', errors='ignore')
except:
return None
# Skip very large text files
if len(content) > 500_000:
return None
cves = extract_cves(content)
platforms = detect_platforms(content, rel_path)
types = detect_types(content, rel_path)
return {
'rel_path': rel_path,
'size_bytes': size,
'cve_refs': json.dumps(cves),
'platforms': json.dumps(platforms),
'types': json.dumps(types),
'cve_list': cves
}
except Exception as e:
return None
def scan_repository(repo_name, repo_path):
"""Scan a repository and return metadata."""
if not os.path.exists(repo_path):
print(f" [SKIP] {repo_name}: path does not exist")
return None
print(f" [SCAN] {repo_name}...")
total_files = 0
total_size = 0
all_cves = set()
file_records = []
scanned_count = 0
for root, dirs, files in os.walk(repo_path):
# Skip .git directory and other build directories
skip_dirs = ['.git', 'node_modules', 'vendor', '__pycache__', 'dist', 'build', 'target', 'bin', 'obj', 'Debug', 'Release', 'x64', 'x86', 'packages', 'venv', 'env', '.venv', '.env']
dirs[:] = [d for d in dirs if d not in skip_dirs]
for file in files:
filepath = Path(root) / file
try:
rel_path = filepath.relative_to(repo_path).as_posix()
except:
rel_path = filepath.as_posix()
result = scan_file(filepath, rel_path)
if result:
total_files += 1
total_size += result['size_bytes']
all_cves.update(result['cve_list'])
file_records.append((repo_name, result))
scanned_count += 1
if scanned_count % 2000 == 0:
print(f" Scanned {scanned_count} files, indexed {total_files}...")
# Limit files per repo for performance
if total_files >= 5000:
print(f" Reached file limit (5000), stopping scan")
break
if total_files >= 5000:
break
print(f" Completed: {total_files} files indexed, {len(all_cves)} CVEs found")
return {
'repo_name': repo_name,
'repo_path': repo_path,
'total_files': total_files,
'total_size': total_size,
'cve_count': len(all_cves),
'cves': list(all_cves),
'file_records': file_records
}
def main():
print("=" * 60)
print("Building Exploit Repository Index")
print("=" * 60)
conn = init_db()
cursor = conn.cursor()
repo_stats = {}
for repo_name, repo_path in REPO_PATHS.items():
result = scan_repository(repo_name, repo_path)
if not result:
continue
# Insert repository
cursor.execute('''
INSERT OR REPLACE INTO repositories (name, path, total_files, total_size_bytes, cve_count)
VALUES (?, ?, ?, ?, ?)
''', (repo_name, repo_path, result['total_files'], result['total_size'], result['cve_count']))
repo_id = cursor.lastrowid
if not repo_id:
cursor.execute('SELECT id FROM repositories WHERE name = ?', (repo_name,))
row = cursor.fetchone()
repo_id = row[0] if row else None
if repo_id:
# Insert file records
for _, frecord in result['file_records']:
cursor.execute('''
INSERT INTO files (repo_id, rel_path, size_bytes, cve_refs, platforms, types)
VALUES (?, ?, ?, ?, ?, ?)
''', (repo_id, frecord['rel_path'], frecord['size_bytes'],
frecord['cve_refs'], frecord['platforms'], frecord['types']))
file_id = cursor.lastrowid
# Insert CVE references
cves = json.loads(frecord['cve_refs'])
for cve in cves:
cursor.execute('''
INSERT INTO cve_references (repo_id, cve_id, file_path)
VALUES (?, ?, ?)
''', (repo_id, cve, frecord['rel_path']))
repo_stats[repo_name] = {
'files': result['total_files'],
'size_mb': round(result['total_size'] / (1024 * 1024), 2),
'cves': result['cve_count']
}
print(f" Files: {result['total_files']}, Size: {round(result['total_size'] / (1024 * 1024), 2)} MB, CVEs: {result['cve_count']}")
conn.commit()
# Print summary
print("\n" + "=" * 60)
print("SUMMARY")
print("=" * 60)
total_files = sum(s['files'] for s in repo_stats.values())
total_size = sum(s['size_mb'] for s in repo_stats.values())
total_cves = sum(s['cves'] for s in repo_stats.values())
print(f"{'Repository':<40} {'Files':>8} {'Size (MB)':>10} {'CVEs':>6}")
print("-" * 68)
for repo_name, stats in sorted(repo_stats.items()):
print(f"{repo_name:<40} {stats['files']:>8} {stats['size_mb']:>10.2f} {stats['cves']:>6}")
print("-" * 68)
print(f"{'TOTAL':<40} {total_files:>8} {total_size:>10.2f} {total_cves:>6}")
# Show top CVEs
cursor.execute('''
SELECT cve_id, COUNT(DISTINCT repo_id) as repo_count, COUNT(*) as file_count
FROM cve_references
GROUP BY cve_id
ORDER BY repo_count DESC, file_count DESC
LIMIT 20
''')
print("\nTop 20 CVEs by repository coverage:")
print("-" * 50)
for row in cursor.fetchall():
print(f" {row[0]}: {row[1]} repos, {row[2]} files")
conn.close()
print(f"\nIndex saved to: {DB_PATH}")
if __name__ == '__main__':
main()