Анализатор логов nginx, записанных в CSV, выводит в виде таблицы статистику запросов под кодам. http://blindage.org/?p=8580
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

118 lines
4.6 KiB

  1. # ---------------------------------------
  2. # Nginx special format analyzer
  3. # Copyright by Vladimir Smagin, 2017
  4. # -----
  5. # Download new version http://git.blindage.org/21h/nginx-csv-log-analyzer
  6. # Any questions? 21h@blindage.org or http://blindage.org
  7. import csv
  8. from terminaltables import AsciiTable
  9. import argparse
  10. import time
  11. # custom nginx log file:
  12. # log_format compression '"$remote_addr","$time_local","$status","$request";';
  13. # Position | Description
  14. # 0 | IP address
  15. # 1 | Time stamp
  16. # 2 | Response code
  17. # 3 | Request [type url protocol]
  18. # timestamp format in logfile
  19. # datetime.datetime.strptime(ts,'%d/%b/%Y:%H:%M:%S %z')
  20. # user's timestamp
  21. # datetime.datetime.strptime(ts,'%d-%m-%Y %H:%M:%S')
  22. cmdLineParser = argparse.ArgumentParser(description='Get statistics from nginx special format log file')
  23. cmdLineParser.add_argument('--log', metavar='logfile', type=str, required=True,
  24. help='Of course, you need to parse something')
  25. cmdLineParser.add_argument('--code', metavar='HTTP_CODE', type=int, help='Show only response code')
  26. cmdLineParser.add_argument('--time', metavar='', type=str, help='Start counting from timestamp DD-MM-YYYY HH:MM:SS')
  27. cmdLineParser.add_argument('--totime', metavar='', type=str, help='Stop counting from timestamp DD-MM-YYYY HH:MM:SS')
  28. cmdLineParser.add_argument('--summ', action='store_true', help='Display fancy summary')
  29. args = cmdLineParser.parse_args()
  30. def summary(userUnixTimeStamp, userUnixTimeStampStop):
  31. with open(args.log) as nginxLogFile:
  32. successCounter = 0
  33. redirectCounter = 0
  34. clientErrorCounter = 0
  35. serverErrorCounter = 0
  36. serverErrorCounter500 = 0
  37. serverErrorCounter502 = 0
  38. allRequests = 0
  39. logStringsIterable = csv.reader(nginxLogFile, delimiter=",", quotechar='"', lineterminator=";")
  40. for logString in logStringsIterable:
  41. # Now parse date if not False
  42. # It's painful
  43. if userUnixTimeStamp is not False:
  44. logUnixTimeStamp = time.strptime(logString[1], '%d/%b/%Y:%H:%M:%S %z')
  45. if userUnixTimeStamp >= logUnixTimeStamp:
  46. continue
  47. if userUnixTimeStampStop < logUnixTimeStamp:
  48. break
  49. allRequests += 1
  50. if '20' in logString[2]: successCounter += 1
  51. if '30' in logString[2]: redirectCounter += 1
  52. if '40' in logString[2]: clientErrorCounter += 1
  53. if '500' in logString[2]: serverErrorCounter500 += 1
  54. if '502' in logString[2]: serverErrorCounter502 += 1
  55. if '50' in logString[2]: serverErrorCounter += 1
  56. # summary
  57. tableData = [
  58. ['HTTP codes', 'Type', 'Count'],
  59. ['All', '', "{:,}".format(allRequests)],
  60. ['20x', 'Success', "{:,}".format(successCounter)],
  61. ['30x', 'Redirection', "{:,}".format(redirectCounter)],
  62. ['40x', 'Client errors', "{:,}".format(clientErrorCounter)],
  63. ['500', 'Server errors', "{:,}".format(serverErrorCounter500)],
  64. ['502', 'Server errors', "{:,}".format(serverErrorCounter502)],
  65. ['50x', 'Server errors', "{:,}".format(serverErrorCounter)]
  66. ]
  67. resultTable = AsciiTable(tableData)
  68. resultTable.inner_heading_row_border = True
  69. resultTable.outer_border = False
  70. resultTable.inner_row_border = False
  71. print(resultTable.table)
  72. def searchCode(httpCode, userUnixTimeStamp, userUnixTimeStampStop):
  73. with open(args.log) as nginxLogFile:
  74. codeCounter = 0
  75. logStringsIterable = csv.reader(nginxLogFile, delimiter=",", quotechar='"', lineterminator=";")
  76. for logString in logStringsIterable:
  77. # Now parse date if not False
  78. # It's painful, yes, I know, it was earlier and now it must feels easier
  79. if userUnixTimeStamp is not False:
  80. logUnixTimeStamp = time.strptime(logString[1], '%d/%b/%Y:%H:%M:%S %z')
  81. if userUnixTimeStamp >= logUnixTimeStamp:
  82. continue
  83. if userUnixTimeStampStop < logUnixTimeStamp:
  84. break
  85. if str(httpCode) in logString[2]:
  86. codeCounter += 1
  87. # print result
  88. print(codeCounter)
  89. # Set start time point
  90. if args.time:
  91. unixTimeStamp = time.strptime(args.time, '%d-%m-%Y %H:%M:%S')
  92. else:
  93. unixTimeStamp = False
  94. # Set stop time point
  95. if args.totime:
  96. unixTimeStampStop = time.strptime(args.totime, '%d-%m-%Y %H:%M:%S')
  97. else:
  98. unixTimeStampStop = False
  99. if args.code:
  100. searchCode(args.code, unixTimeStamp, unixTimeStampStop)
  101. if args.summ:
  102. summary(unixTimeStamp, unixTimeStampStop)