web_crawler_detection/access.lua at master · SSbug696/web_crawler_detection · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
local _O = {}

function ipInt(ip)
  local ip_int = 0
  local iter = string.gmatch(ip, "(%d+)")
  local count_octet = 4

  for k in iter do
    local octet = tonumber(k)

    ip_int = ip_int + (octet * math.pow(256, count_octet - 1))

    count_octet = count_octet - 1
  end

  return ip_int
end

local white_list = {
--  {2886795264, 2886795266, 0},
  {1680556032, 1680560127, 2},
  {1680560128, 1680560383, 2},
  {1680560384, 1680560639, 2},
  {1680560640, 1680560895, 2},
  {1680560896, 1680561151, 2}
}

function isNetAccess(ip)
  local exist_flag = false
  -- Detect crawler by header
  -- local service_id = available_header[header]
  local min = 0
  local max = 0
  local list_access_params
  local ip_as_number = ipInt(ip)

  for index in ipairs(white_list) do
    -- Get record
    list_access_params = white_list[index]

    -- If ID range equal to ckeck address by current range
    min = list_access_params[1]
    max = list_access_params[2]

    -- Check range
    if min <= ip_as_number and max >= ip_as_number then
      return true
    end
  end

  return exist_flag
end

--- Method detection search engines crawlers
function accessUA(user_agent)
	local available_uagents = {
        	'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
        	'Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
        	'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
        	'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
        	'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
	}

	for i in pairs(available_uagents) do
		if available_uagents[i] == user_agent then
			return true
		end
	end

	return false
end


function writeLog(log_msg)
	local io = require "io"
	local file=io.open('etc/nginx/lualib/log/log.txt', 'a+')
	file:write(log_msg , '\n')
	file:close()
end


function _O.sendData(data, uri, remote_addr)
	local str = ''
	local user_agent = ''
	local host = ''
	local uri_request

	for k, v in pairs(data) do
		if k == 'user-agent' then
			user_agent = v

			if accessUA(user_agent) == false then
				return 'false'
			end
		end

		if k == 'host' then
			host = v
			uri_request = uri
		end
	end

	local redis = require "resty.redis"
	local red = redis:new()

       	red:set_timeout(1000) -- 1 sec

	local ok, err = red:connect('127.0.0.1', 6379)
	if err then
		 writeLog('Error of redis connection:' .. err)
	end

	ok, err = red:set('botstat:' .. host ..':' .. uri, os.time() )
	if err then
		writeLog('Error of call method from redis:' .. err)
	end

	-- return   host .. '  ' .. user_agent .. '  host:' .. uri .. ' ' .. tostring(isNetAccess(remote_addr)) .. '  ipint:' .. tostring(ipInt(remote_addr)) .. ' '  .. remote_addr
	if isNetAccess(remote_addr) and accessUA(user_agent) then
		return true
	else
		return false
	end
end

return _O