Skip to content

Commit 5b8a2fa

Browse files
stamparmstamparm
committed
Doing some cleaning for web search requests
1 parent da21091 commit 5b8a2fa

5 files changed

Lines changed: 28 additions & 22 deletions

File tree

‎lib/core/settings.py‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,14 @@
1212
import re
1313
import string
1414
import sys
15-
import time
1615

1716
from lib.core.enums import DBMS
1817
from lib.core.enums import DBMS_DIRECTORY_NAME
1918
from lib.core.enums import OS
2019
from thirdparty import six
2120

2221
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
23-
VERSION = "1.10.10.16"
22+
VERSION = "1.10.10.17"
2423
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
2524
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
2625
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -204,12 +203,6 @@
204203
("", 200),
205204
)
206205

207-
# Regular expression used for extracting results from Google search
208-
GOOGLE_REGEX = r"webcache\.googleusercontent\.com/search\?q=cache:[^:]+:([^+]+)\+&amp;cd=|url\?\w+=((?![^>]+webcache\.googleusercontent\.com)http[^>]+)&(sa=U|rct=j)"
209-
210-
# Google Search consent cookie
211-
GOOGLE_CONSENT_COOKIE = "CONSENT=YES+shp.gws-%s-0-RC1.%s+FX+740" % (time.strftime("%Y%m%d"), "".join(random.sample(string.ascii_lowercase, 2)))
212-
213206
# Regular expression used for extracting results from DuckDuckGo search
214207
DUCKDUCKGO_REGEX = r'<a class="result__url" href="(htt[^"]+)'
215208

‎lib/parse/cmdline.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,8 @@ def cmdLineParser(argv=None):
138138
target.add_argument("-r", dest="requestFile",
139139
help="Load HTTP request from a file")
140140

141-
target.add_argument("-g", dest="googleDork",
142-
help="Process Google dork results as target URLs")
141+
target.add_argument("-g", "--dork", dest="googleDork", metavar="DORK",
142+
help="Process search engine dork results as target URLs")
143143

144144
target.add_argument("-c", dest="configFile",
145145
help="Load options from a configuration INI file")
@@ -736,8 +736,8 @@ def cmdLineParser(argv=None):
736736
general.add_argument("--fresh-queries", dest="freshQueries", action="store_true",
737737
help="Ignore query results stored in session file")
738738

739-
general.add_argument("--gpage", dest="googlePage", type=int,
740-
help="Use Google dork results from specified page number")
739+
general.add_argument("--gpage", dest="googlePage", type=int, metavar="PAGE",
740+
help="Use dork results from specified page number")
741741

742742
general.add_argument("--har", dest="harFile",
743743
help="Log all HTTP traffic into a HAR file")

‎lib/utils/search.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@
2626
from lib.core.settings import BING_REGEX
2727
from lib.core.settings import DUCKDUCKGO_REGEX
2828
from lib.core.settings import DUMMY_SEARCH_USER_AGENT
29-
from lib.core.settings import GOOGLE_CONSENT_COOKIE
30-
from lib.core.settings import GOOGLE_REGEX
3129
from lib.core.settings import HTTP_ACCEPT_ENCODING_HEADER_VALUE
3230
from lib.core.settings import SEARCH_ABUSE_MARKERS
3331
from lib.core.settings import UNICODE_ENCODING
@@ -91,19 +89,20 @@ def _search(dork):
9189
requestHeaders = {
9290
HTTP_HEADER.USER_AGENT: dict(conf.httpHeaders).get(HTTP_HEADER.USER_AGENT, DUMMY_SEARCH_USER_AGENT),
9391
HTTP_HEADER.ACCEPT_ENCODING: HTTP_ACCEPT_ENCODING_HEADER_VALUE,
94-
HTTP_HEADER.COOKIE: GOOGLE_CONSENT_COOKIE,
9592
}
9693

9794
gpage = conf.googlePage if conf.googlePage > 1 else 1
9895
logger.info("using search result page #%d" % gpage)
9996

10097
encoded = urlencode(dork, convall=True)
10198

102-
# Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (DuckDuckGo currently being the only consistently scrapeable one)
99+
# Note: DuckDuckGo's first result page holds 10 entries, while each subsequent one holds 15 (hence the non-linear offset)
100+
ddgOffset = 0 if gpage == 1 else 10 + (gpage - 2) * 15
101+
102+
# Note: (name, url, POST data, regex, regex flags, match->link). Ordered by current scraping reliability; tried in turn until one yields results (Google is not in here as it serves results only to JavaScript capable clients)
103103
engines = (
104-
("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, (gpage - 1) * 30), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&amp;", "&")),
104+
("DuckDuckGo", "https://html.duckduckgo.com/html/", "q=%s&s=%d" % (encoded, ddgOffset), DUCKDUCKGO_REGEX, re.I | re.S, lambda match: match.group(1).replace("&amp;", "&")),
105105
("Bing", "https://www.bing.com/search?q=%s&first=%d" % (encoded, (gpage - 1) * 10 + 1), None, BING_REGEX, re.I | re.S, lambda match: decodeBase64(match.group(1), binary=False) if match.group(1) else match.group(2).replace("&amp;", "&")),
106-
("Google", "https://www.google.com/search?q=%s&num=100&hl=en&complete=0&safe=off&filter=0&btnG=Search&start=%d" % (encoded, (gpage - 1) * 100), None, GOOGLE_REGEX, re.I, lambda match: match.group(1) or match.group(2)),
107106
)
108107

109108
for name, url, data, regex, flags, extract in engines:

‎sqlmap.conf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ bulkFile =
2525
# Example (file content): POST /login.jsp HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/4.0\n\nuserid=joe&password=guessme
2626
requestFile =
2727

28-
# Rather than providing a target URL, let Google return target
29-
# hosts as result of your Google dork expression. For a list of Google
28+
# Rather than providing a target URL, let the search engine return target
29+
# hosts as result of your dork expression. For a list of
3030
# dorks see Google Hacking Database at
3131
# https://www.exploit-db.com/google-hacking-database
3232
# Example: +ext:php +inurl:"&id=" +intext:"powered by "
@@ -797,7 +797,7 @@ forms = False
797797
# Valid: True or False
798798
freshQueries = False
799799

800-
# Use Google dork results from specified page number.
800+
# Use dork results from specified page number.
801801
# Valid: integer
802802
# Default: 1
803803
googlePage = 1

‎tests/test_dork_search.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,13 @@ def _pages(self, *pages):
4242
"""Patches _fetch to serve `pages` in engine order (None meaning no reply)"""
4343

4444
served = list(pages)
45-
smod._fetch = lambda *args, **kwargs: served.pop(0) if served else None
45+
self.requested = []
46+
47+
def fetch(url, headers, data=None):
48+
self.requested.append((url, data))
49+
return served.pop(0) if served else None
50+
51+
smod._fetch = fetch
4652

4753
def test_duckduckgo_results(self):
4854
self._pages(DUCKDUCKGO_PAGE)
@@ -52,6 +58,14 @@ def test_bing_fallback_decodes_redirect(self):
5258
self._pages(None, BING_PAGE)
5359
self.assertEqual(smod._search("dork"), ["https://www.example.com/a.php?id=1"])
5460

61+
def test_duckduckgo_paging_offset(self):
62+
# Note: first result page holds 10 entries, each subsequent one 15 (and not the uniform 30 as previously used)
63+
for gpage, offset in ((1, 0), (2, 10), (3, 25), (4, 40)):
64+
conf.googlePage = gpage
65+
self._pages(DUCKDUCKGO_PAGE)
66+
smod._search("dork")
67+
self.assertEqual(self.requested[0][1], "q=dork&s=%d" % offset)
68+
5569
def test_abuse_page_switches_proxy(self):
5670
# Note: without '--proxy-list' there is nothing to switch to, hence no exception
5771
for marker in SEARCH_ABUSE_MARKERS:

0 commit comments

Comments
 (0)