2024年6月

FOFA搜索:

body="get all proxy from proxy pool"

可以得到很多爬虫维护的代理站点:

http://113.235.121.92:5010/
http://112.74.98.149:5010/
http://159.75.88.180:5010/
http://101.200.37.197:5010/
https://49.235.191.186/
...……

拉取一个:

http://113.235.121.92:5010/all/

可得:

[{"anonymous":"","check_count":60,"fail_count":0,"https":true,"last_status":true,"last_time":"2024-06-26 14:47:38","proxy":"47.96.176.130:59394","region":"\u4e2d\u56fd \u6d59\u6c5f \u676d\u5dde \u963f\u91cc\u4e91/\u7535\u4fe1/\u8054\u901a/\u79fb\u52a8/\u94c1\u901a/\u6559\u80b2\u7f51","source":"freeProxy11"},{"anonymous":"","check_count":2,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:42","proxy":"154.203.132.49:8080","region":"\u4e2d\u56fd \u9999\u6e2f  cloudinnovation.org","source":"freeProxy11"},{"anonymous":"","check_count":5,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:38","proxy":"183.60.141.41:443","region":"\u4e2d\u56fd \u5e7f\u4e1c \u4e1c\u839e \u7535\u4fe1","source":"freeProxy11"},{"anonymous":"","check_count":44,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:39","proxy":"178.48.68.61:18080","region":"\u5308\u7259\u5229 \u5308\u7259\u5229  upc.hu","source":"freeProxy11"},{"anonymous":"","check_count":13,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:45:44","proxy":"50.62.183.223:80","region":"\u7f8e\u56fd \u4e9a\u5229\u6851\u90a3\u5dde \u51e4\u51f0\u57ce godaddy.com","source":"freeProxy11"},{"anonymous":"","check_count":163,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:37","proxy":"116.63.129.202:6000","region":"\u4e2d\u56fd \u8d35\u5dde \u8d35\u9633 \u7535\u4fe1/\u8054\u901a/\u79fb\u52a8","source":"freeProxy11"},{"anonymous":"","check_count":1,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:45:51","proxy":"120.197.40.219:9002","region":"\u4e2d\u56fd \u5e7f\u4e1c \u5e7f\u5dde \u79fb\u52a8","source":"freeProxy11/freeProxy03"},{"anonymous":"","check_count":14,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:41","proxy":"62.236.76.83:8085","region":"\u82ac\u5170 \u65b0\u5730\u533a \u8d6b\u5c14\u8f9b\u57fa dna.fi","source":"freeProxy11"},{"anonymous":"","check_count":3,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:45:44","proxy":"142.93.142.36:9122","region":"\u8377\u5170 \u5317\u8377\u5170\u7701 \u963f\u59c6\u65af\u7279\u4e39 digitalocean.com","source":"freeProxy11"},{"anonymous":"","check_count":6,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:43","proxy":"154.203.132.55:8090","region":"\u4e2d\u56fd \u9999\u6e2f  cloudinnovation.org","source":"freeProxy11"},{"anonymous":"","check_count":4,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:37","proxy":"223.113.80.158:9091","region":"\u4e2d\u56fd \u6c5f\u82cf \u5f90\u5dde \u79fb\u52a8","source":"freeProxy11"},{"anonymous":"","check_count":46,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:39","proxy":"112.19.241.37:19999","region":"\u4e2d\u56fd \u56db\u5ddd \u6210\u90fd \u79fb\u52a8","source":"freeProxy11"},{"anonymous":"","check_count":2,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:37","proxy":"51.89.73.162:80","region":"\u5fb7\u56fd \u9ed1\u68ee\u5dde \u5170\u6cb3\u7554\u6797\u5821 ovh.com","source":"freeProxy11"},{"anonymous":"","check_count":5,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:43","proxy":"154.203.132.55:8080","region":"\u4e2d\u56fd \u9999\u6e2f  cloudinnovation.org","source":"freeProxy11"},{"anonymous":"","check_count":7,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:37","proxy":"60.12.168.114:9002","region":"\u4e2d\u56fd \u6d59\u6c5f \u53f0\u5dde \u8054\u901a","source":"freeProxy03"},{"anonymous":"","check_count":2,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:40","proxy":"8.209.253.237:8080","region":"\u65e5\u672c \u4e1c\u4eac\u90fd \u4e1c\u4eac \u963f\u91cc\u4e91","source":"freeProxy07"},{"anonymous":"","check_count":9,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:47:37","proxy":"111.160.204.146:9091","region":"\u4e2d\u56fd \u5929\u6d25 \u5929\u6d25 \u8054\u901a","source":"freeProxy11"},{"anonymous":"","check_count":3,"fail_count":0,"https":false,"last_status":true,"last_time":"2024-06-26 14:45:54","proxy":"161.34.40.109:3128","region":"\u65e5\u672c \u65e5\u672c  sphere.ne.jp","source":"freeProxy11"},{"anonymous":"","check_count":4,"fail_count":0,"https":true,"last_status":true,"last_time":"2024-06-26 14:45:48","proxy":"160.248.93.84:3128","region":"\u65e5\u672c \u65e5\u672c  sphere.ne.jp","source":"freeProxy11"}]

验证

#!/bin/bash

# 代理列表
proxies=(
  "47.96.176.130:59394"
  "178.48.68.61:18080"
  "91.148.127.162:8080"
  "50.62.183.223:80"
  "116.63.129.202:6000"
  "120.197.40.219:9002"
  "212.127.93.185:8081"
  "154.203.132.55:8090"
  "142.93.142.36:9122"
  "223.113.80.158:9091"
  "112.19.241.37:19999"
  "51.89.73.162:80"
  "154.203.132.55:8080"
  "49.13.9.253:80"
  "79.110.196.145:8081"
  "60.12.168.114:9002"
  "111.160.204.146:9091"
  "91.148.126.189:8080"
  "161.34.40.109:3128"
  "160.248.93.84:3128"
  "8.219.97.248:80"
  "103.153.154.6:80"
)

# 目标URL
url="http://www.baidu.com"

# 检查代理是否有效
for proxy in "${proxies[@]}"; do
  # 通过代理发送请求
  response=$(curl -s -o /dev/null -w "%{http_code}" -x "$proxy" "$url")
  
  # 检查响应状态码
  if [ "$response" -eq 200 ]; then
    echo "Proxy $proxy is valid."
  else
    echo "Proxy $proxy is invalid."
  fi
done

结果:

Proxy 47.96.176.130:59394 is valid.
Proxy 178.48.68.61:18080 is valid.
Proxy 91.148.127.162:8080 is valid.
Proxy 50.62.183.223:80 is valid.
Proxy 116.63.129.202:6000 is valid.
Proxy 120.197.40.219:9002 is valid.
Proxy 212.127.93.185:8081 is valid.
Proxy 154.203.132.55:8090 is valid.
Proxy 142.93.142.36:9122 is valid.
Proxy 223.113.80.158:9091 is valid.
Proxy 112.19.241.37:19999 is valid.
Proxy 51.89.73.162:80 is valid.
Proxy 154.203.132.55:8080 is valid.
Proxy 49.13.9.253:80 is invalid.
Proxy 79.110.196.145:8081 is valid.
Proxy 60.12.168.114:9002 is valid.
Proxy 111.160.204.146:9091 is valid.
Proxy 91.148.126.189:8080 is invalid.
Proxy 161.34.40.109:3128 is invalid.
Proxy 160.248.93.84:3128 is valid.
Proxy 8.219.97.248:80 is valid.
Proxy 103.153.154.6:80 is valid.

总结

  1. 可以通过脚本自动获取并验证更新代理节点
  2. 还有更多的搜索方法:

      protocol=="socks5" && "Version:5 Method:No Authentication(0x00)" && after="1900-01-01" && country="CN"
     "“89e3175.js”" 或title="代理池网页管理界面"

常用的标准Linux环境里都有Systemd提供后台服务进程管理及保活,但是在一些特殊的运行环境还是需要简易的解决方案。

比如在serv00服务器freeBSD Linux环境下,可以写这个这样的脚本:

check_and_start_process.sh

#!/bin/bash

# Define process name and start command
PROCESS_NAME="command"
START_COMMAND="/path/to/your/command"

# Check if the process is running
if ! pgrep -f "$PROCESS_NAME" > /dev/null
then
    echo "Process $PROCESS_NAME is not running, starting it now..."
    # Start the process
    $START_COMMAND &
    if [ $? -eq 0 ]; then
        echo "Process $PROCESS_NAME started successfully."
    else
        echo "Failed to start process $PROCESS_NAME."
    fi
else
    echo "Process $PROCESS_NAME is already running."
fi

修改脚本中的PROCESS_NAMESTART_COMMAND值,并增加运行权限:

chmod +x check_and_start_process.sh

接着添加定时任务:

crontab -e

每5分钟检测一次:

*/5 * * * * /path/to/check_and_start_process.sh

提示
在serv00服务器中,定时任务执行后如果会触发email,则改用:

*/5 * * * * /path/to/check_and_start_process.sh > /dev/null 2>&1

参考网上资料梳理的Cloudflare IP地址归属地信息,具体准确性待考证:

{
  "PH": [
    "172.69.184.0/22",
    "162.158.136.0/22",
    "162.158.228.0/22"
  ],
  "LT": [
    "172.69.188.0/22"
  ],
  "HK": [
    "103.22.202.0/24",
    "103.22.203.0/24",
    "108.162.222.0/24",
    "108.162.223.0/24",
    "162.158.176.0/24",
    "162.158.177.0/24",
    "162.158.178.0/24",
    "162.158.179.0/24",
    "172.68.248.0/22",
    "172.69.96.0/22",
    "172.69.176.0/22",
    "173.245.62.0/24"
  ],
  "TW": [
    "103.31.4.0/23",
    "162.158.224.0/22",
    "162.158.240.0/22"
  ],
  "MO": [
    "172.69.72.0/22"
  ],
  "KH": [
    "172.69.80.0/22"
  ],
  "CA": [
    "108.162.208.0/24",
    "108.162.240.0/24",
    "108.162.241.0/24",
    "162.158.144.0/24",
    "162.158.145.0/24",
    "162.158.146.0/24",
    "162.158.147.0/24",
    "172.69.156.0/22",
    "172.69.212.0/22",
    "172.69.216.0/22"
  ],
  "US-LA": [
    "108.162.214.0/24",
    "108.162.215.0/24",
    "141.101.72.0/24",
    "162.158.56.0/22",
    "172.68.44.0/22",
    "172.68.208.0/22",
    "172.69.32.0/22",
    "173.245.48.0/24"
  ],
  "US-CH": [
    "108.162.216.0/24",
    "108.162.217.0/24",
    "141.101.73.0/24",
    "162.158.72.0/22",
    "172.68.56.0/24",
    "172.68.57.0/24",
    "172.68.58.0/24",
    "172.68.59.0/24"
  ],
  "US-PH": [
    "162.158.140.0/24",
    "162.158.141.0/24",
    "162.158.142.0/24",
    "162.158.143.0/24"
  ],
  "US-NJ": [
    "108.162.218.0/24",
    "108.162.219.0/24",
    "162.158.60.0/22"
  ],
  "US-NY": [
    "173.245.52.0/24"
  ],
  "US-DA": [
    "108.162.220.0/24",
    "108.162.221.0/24",
    "141.101.74.0/24",
    "172.69.64.0/21"
  ],
  "US-SJ": [
    "162.158.252.0/22",
    "172.68.80.0/22",
    "172.68.132.0/22",
    "172.68.140.0/22",
    "172.68.188.0/22",
    "172.69.16.0/22"
  ],
  "US-LV": [
    "162.158.244.0/22"
  ],
  "JP-TK": [
    "108.162.226.0/24",
    "108.162.227.0/24",
    "103.22.200.0/23",
    "162.158.4.0/22",
    "162.158.116.0/22",
    "172.69.108.0/22"
  ],
  "JP-OS": [
    "172.69.152.0/22"
  ],
  "FR": [
    "108.162.228.0/24",
    "141.101.66.0/24",
    "141.101.67.0/24",
    "141.101.68.0/24",
    "141.101.69.0/24",
    "108.162.229.0/24",
    "141.101.88.0/22",
    "162.158.192.0/24",
    "162.158.193.0/24",
    "162.158.194.0/24",
    "162.158.195.0/24",
    "172.69.224.0/22"
  ],
  "US-MI": [
    "162.158.120.0/24",
    "162.158.121.0/24",
    "162.158.122.0/24",
    "162.158.123.0/24",
    "108.162.210.0/24",
    "108.162.211.0/24",
    "108.162.212.0/24",
    "108.162.213.0/24"
  ],
  "US-AT": [
    "108.162.236.0/24",
    "108.162.237.0/24",
    "108.162.238.0/24",
    "162.158.124.0/22",
    "162.158.184.0/24",
    "162.158.185.0/24",
    "162.158.186.0/24",
    "162.158.187.0/24"
  ],
  "US-DN": [
    "172.68.32.0/22"
  ],
  "US-SL": [
    "172.68.36.0/22"
  ],
  "US-BO": [
    "172.68.52.0/22"
  ],
  "US-SEA": [
    "108.162.243.0/24",
    "108.162.244.0/24",
    "108.162.245.0/24",
    "108.162.246.0/24",
    "162.158.104.0/24",
    "162.158.105.0/24",
    "162.158.106.0/24",
    "162.158.107.0/24"
  ],
  "US-AH": [
    "162.158.76.0/22",
    "172.68.64.0/22",
    "172.69.60.0/22",
    "173.245.54.0/24",
    "188.114.104.0/24"
  ],
  "US-KC": [
    "172.68.148.0/22"
  ],
  "US-DC": [
    "172.68.204.0/22"
  ],
  "US-PN": [
    "172.68.172.0/22"
  ],
  "US-HO": [
    "172.69.168.0/22"
  ],
  "US-SC": [
    "172.69.40.0/22"
  ],
  "US-SD": [
    "172.68.228.0/22"
  ],
  "US-PA": [
    "172.68.68.0/22"
  ],
  "US-TN": [
    "172.68.72.0/22"
  ],
  "US-TB": [
    "172.68.76.0/22"
  ],
  "US-OM": [
    "172.68.88.0/22"
  ],
  "US-MC": [
    "172.69.4.0/22"
  ],
  "US-PI": [
    "172.69.48.0/22"
  ],
  "US-IN": [
    "172.69.88.0/22"
  ],
  "US-SA": [
    "172.69.140.0/22"
  ],
  "US-TH": [
    "172.69.180.0/22"
  ],
  "AU-ME": [
    "108.162.250.0/24",
    "108.162.251.0/24",
    "141.101.64.0/24",
    "141.101.65.0/24",
    "162.158.240.0/24",
    "162.158.241.0/24",
    "162.158.242.0/24",
    "162.158.243.0/24"
  ],
  "AU-SY": [
    "103.22.204.0/22",
    "108.162.246.0/24",
    "108.162.247.0/24",
    "162.158.214.0/24",
    "162.158.215.0/24",
    "162.158.216.0/24",
    "162.158.217.0/24"
  ],
  "SG": [
    "103.31.8.0/22",
    "108.162.192.0/24",
    "108.162.193.0/24",
    "108.162.194.0/24",
    "108.162.195.0/24",
    "108.162.202.0/24",
    "108.162.203.0/24",
    "162.158.18.0/24",
    "162.158.19.0/24",
    "172.68.146.0/22",
    "172.69.24.0/22"
  ],
  "ZA": [
    "162.158.144.0/22"
  ],
  "BR-SP": [
    "108.162.236.0/24",
    "108.162.237.0/24",
    "108.162.238.0/24",
    "108.162.239.0/24"
  ],
  "BR-RJ": [
    "162.158.252.0/22"
  ]
}