Browse Source

bugfixes and html output

master
Vladimir Smagin 2 years ago
parent
commit
aff320f4cc
3 changed files with 97 additions and 36 deletions
  1. +3
    -0
      .gitignore
  2. +29
    -0
      report.html
  3. +65
    -36
      request_checker.py

+ 3
- 0
.gitignore View File

@@ -0,0 +1,3 @@
.idea
config.py
__pycache__

+ 29
- 0
report.html View File

@@ -0,0 +1,29 @@
<h1>Request checker report</h1> Started 2018-06-05 04:37:26.934369
<table width='100%' border='1'>
<tr>
<td> https://online.renins.com/buy/auto/#edit/new/data </td>
<td> <strong>leadslabpixels.net(.*)\/tm.js\?id</strong><br>https://leadslabpixels.net/tm//tm.js?id=8bb303bd-0ac3-474e-ba6c-e3da7b0cdd40<br><strong>leadslabpixels.net(.*)\/tag_manager.js</strong><br>https://leadslabpixels.net/tm/8bb303bd-0ac3-474e-ba6c-e3da7b0cdd40/tag_manager.js<br> </td></tr>
<tr>
<td> http://nevatowers.ru </td>
<td> <strong>leadslabpixels.net(.*)\/tm.js\?id</strong><br>http://leadslabpixels.net/tm//tm.js?id=4A8C1E55-2739-4B13-8F24-DC50133B1FB0<br><strong>leadslabpixels.net(.*)\/tag_manager.js</strong><br>http://leadslabpixels.net/tm/4A8C1E55-2739-4B13-8F24-DC50133B1FB0/tag_manager.js<br><strong>leadslabpixels.net\/Pixel\/studying\?idClient=16</strong><br>http://leadslabpixels.net/Pixel/studying?idClient=16&idCampaign=245&password=06U1peMZ&type=1&userData=$USERDATA<br><strong>leadslabpixels.net\/Pixel\/studying\?idClient=16</strong><br>http://leadslabpixels.net/Pixel/studying?idClient=16&idCampaign=245&password=06U1peMZ&type=2&userData=7d9eaf21-683f-11e8-a9cb-024299c3a11a<br> </td></tr>
<tr>
<td> http://novogradpavlino.ru </td>
<td> <strong>leadslabpixels.net(.*)\/tm.js\?id</strong><br>https://leadslabpixels.net/tm//tm.js?id=29a6d910-75ec-42e1-9ff4-a01e5f4cf0b9<br><strong>leadslabpixels.net(.*)\/tag_manager.js</strong><br>https://leadslabpixels.net/tm/29a6d910-75ec-42e1-9ff4-a01e5f4cf0b9/tag_manager.js<br><strong>leadslabpixels.net\/Pixel\/studying\?idClient=16</strong><br>https://leadslabpixels.net/Pixel/studying?idClient=16&idCampaign=318&password=kS3t80RC&type=1&userData=$USERDATA<br> </td></tr>
<tr>
<td> http://m.uprimskiy.ru/ </td>
<td> <strong>globalex.ru\/pixel\/1.gif\?campaign=dcn1</strong><br>http://globalex.ru/pixel/1.gif?campaign=dcn1<br><strong>rupertino.ru\/code.php\?code\=</strong><br>https://rupertino.ru/code.php?code=XFZDGE5SWkxDU0ZMVEcfQEY=&id1=6071&id2=435650<br><strong>leadslabpixels.net(.*)\/tm.js\?id</strong><br>http://leadslabpixels.net/tm//tm.js?id=311C121A-0740-48C4-BB8E-CE8CE911F110<br><strong>leadslabpixels.net(.*)\/tag_manager.js</strong><br>http://leadslabpixels.net/tm/311C121A-0740-48C4-BB8E-CE8CE911F110/tag_manager.js<br> </td></tr>
<tr>
<td> http://www.autolight.ru/ </td>
<td> </td></tr>
<tr>
<td> http://sales.mercedes-izmaylovo.ru </td>
<td> <strong>rupertino.ru\/code.php\?code\=</strong><br>https://rupertino.ru/code.php?code=XFZDGE1BVVtaU0deSlxEQh1ARw==&id1=1801&id2=161652<br><strong>leadslabpixels.net(.*)\/tm.js\?id</strong><br>http://leadslabpixels.net/tm//tm.js?id=5be04d1a-f920-412a-96a2-dce712ae5489<br><strong>leadslabpixels.net(.*)\/tag_manager.js</strong><br>http://leadslabpixels.net/tm/5be04d1a-f920-412a-96a2-dce712ae5489/tag_manager.js<br> </td></tr>
<tr>
<td> http://100lichny.ru </td>
<td> </td></tr>
<tr>
<td> http://skolkovskiy.ru/ </td>
<td> <strong>leadslabpixels.net(.*)\/tm.js\?id</strong><br>http://leadslabpixels.net/tm//tm.js?id=311C121A-0740-48C4-BB8E-CE8CE911F110<br><strong>leadslabpixels.net(.*)\/tag_manager.js</strong><br>http://leadslabpixels.net/tm/311C121A-0740-48C4-BB8E-CE8CE911F110/tag_manager.js<br><strong>leadslabpixels.net\/Pixel\/studying\?idClient=16</strong><br>http://leadslabpixels.net/Pixel/studying?idClient=16&idCampaign=115&password=2B0RaAPg&type=1&userData=$USERDATA<br><strong>leadslabpixels.net\/Pixel\/studying\?idClient=16</strong><br>http://leadslabpixels.net/Pixel/studying?idClient=16&idCampaign=17&password=nLIWD9t8&type=1&userData=$USERDATA<br><strong>leadslabpixels.net\/Pixel\/studying\?idClient=16</strong><br>http://leadslabpixels.net/Pixel/studying?idClient=16&idCampaign=115&password=2B0RaAPg&type=1&userData=$USERDATA<br> </td></tr>
<tr>
<td> http://dompokolenie.ru </td>
Splash server connection error

+ 65
- 36
request_checker.py View File

@@ -5,10 +5,13 @@
'''

import requests
import re
import mailer
from urllib.parse import quote
import json
import config
import sys, os, signal
import datetime

class C:
HEADER = '\033[95m'
@@ -20,9 +23,12 @@ class C:
BOLD = '\033[1m'
UNDERLINE = '\033[4m'

def sendmail(subj, body):
def sendmail(a, b):
pass

def sendmail2(subj, body):
message = mailer.Message(From='robot@blindage.org',
To='21h@blindage.org')
To=config.mailaddr)
message.Subject = subj
message.Html = """<p>Urgent report message<br>
%s""" % body
@@ -33,44 +39,67 @@ def sendmail(subj, body):
except:
print(C.FAIL+"Can't send message to sysadmin!"+C.ENDC)

for pageUrl in config.searchURLs:
try:
print(C.HEADER+"Checking", pageUrl+C.ENDC)
if (len(sys.argv)>1) and (sys.argv[1] == '-html'):
useHTMLTags = True
else:
useHTMLTags = False

try:
print(requests.get(config.searchServer + "/_ping").text)
print(requests.post(config.searchServer + "/_gc").text)
except:
print(C.FAIL + "Splash server is down. It's time to panic!" + C.ENDC)
break
searchUrls = config.searchForRequests['urls']
searchRegexps = config.searchForRequests['regexps']

pageUrlRequest = config.searchServer+"/render.har?url="+ quote(pageUrl) + ""
print(pageUrlRequest)
r = requests.get(pageUrlRequest)
answer = r.json()
if r.status_code is 200:
scriptNotFound = True
for requestUrl in answer['log']['entries']:
url = requestUrl['response']['url']
for uSFR, aSFR in config.searchForRequests.items():
if useHTMLTags:
print('<h1>Request checker report</h1>', 'Started', datetime.datetime.now())
print("<table width='100%' border='1'>")
else:
print(datetime.datetime.now())

for searchUrl in searchUrls:
if useHTMLTags:
print('<tr>')
#очистить кеш, проверить состояние.
try:
r = requests.get(config.searchServer + "/_ping").text
#print("\t",r)
r = requests.post(config.searchServer + "/_gc").text
#print("\t",r)
except:
print(C.FAIL + "Splash server is down. It's time to panic!" + C.ENDC)
sys.exit(1)

if uSFR in url:
for scriptName in aSFR:
if scriptName in url:
scriptNotFound = False
print(C.OKGREEN + "Found request match", C.OKBLUE + url, uSFR, scriptName + C.ENDC)
try:
if useHTMLTags:
print('<td>', searchUrl, '</td>')
else:
print(C.HEADER+"Checking URL", searchUrl+C.ENDC)
pageUrlRequest = config.searchServer+"/render.har?url="+ quote(searchUrl)# + "&wait=30"
r = requests.get(pageUrlRequest)
except:
print(C.WARNING + "Splash server connection error" + C.ENDC)
sys.exit(1)

if scriptNotFound:
print(C.FAIL + "No matches found" + C.ENDC)
sendmail('URL ' + pageUrl + ' doesn\'t contain needed script request',"<p>URL " + pageUrl +
"</p><p>No matches found with templates</p><p>" + r.text + "</p>")
if int(r.status_code) is 200:
answer = r.json()
htmlString = ''
for requestUrl in answer['log']['entries']:
url = requestUrl['response']['url']
for searchRegexp in searchRegexps:
if (re.search(searchRegexp, url)):
if useHTMLTags:
htmlString += '<strong>'+searchRegexp+'</strong><br>'+url+'<br>'
else:
print(C.OKGREEN + "Found request match", "\n\t", C.OKBLUE + url,
"\n\t", searchRegexp + C.ENDC)
else:
if useHTMLTags:
htmlString = '<font color=red>',r.status_code, r.text,'</font>'
else:
print(C.WARNING + "Response code is not 200" + C.ENDC)
sendmail('URL ' + pageUrl + ' returned wrong response code', "<p>URL " + pageUrl +
"</p><p>Response code is not 200</p><p>" + r.text + "</p>")
print(C.WARNING + "Response code is not 200, got " + str(r.status_code) + C.ENDC)

except ValueError:
print(ValueError)
print(C.WARNING + "Couldn't connect to server" + C.ENDC)
sendmail('URL '+ pageUrl+' inacessible', "<p>Status "+str(r.status_code)+"</p><p>Answer<br><br>"+r.text+"</p>")
if useHTMLTags:
print('<td>', htmlString,"</td></tr>")
sys.stdout.flush()

if useHTMLTags:
print("</table>")
print('Finished:', datetime.datetime.now())

Loading…
Cancel
Save