Skip to content

fix: online source crawl failure #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
yinhang-joy wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: online source crawl failure #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"name": "shadow-reader",
"displayName": "shadow reader",
"description": "摸鱼划水看书，十分隐蔽",
"version": "0.8.2",
"version": "0.8.3",
"publisher": "rainbroadcast",
"engines": {
"vscode": "^1.54.0"
Expand Down Expand Up @@ -31,10 +31,10 @@
},
"shadowReader.onlineBookURL": {
"type": "string",
"default": "https://www.biqugee6.com",
"default": "https://www.biquge7.xyz",
"enum": [
"https://www.caimoge.net",
"https://www.biqugee6.com"
"https://www.caimoge.com",
"https://www.biquge7.xyz"
],
"enumDescriptions": [
"采墨阁",
Expand Down
4 changes: 2 additions & 2 deletions src/const.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export const CrawelerDomains = new Map<string, string>([
["biquURL", "https://www.biqugee6.com"],
["caimoURL", "https://www.caimoge.net"],
["biquURL", "https://www.biquge7.xyz"],
["caimoURL", "https://www.caimoge.com"],
]);
93 changes: 40 additions & 53 deletions src/crawler/biqu.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,75 +1,55 @@
import cheerioModule = require("cheerio");
import axios from "axios";
import iconv = require('iconv-lite');
import https = require('https');
import { window } from "vscode";
import { Craweler } from "./interface";

const ignoreSSL = axios.create({
httpsAgent: new https.Agent({
rejectUnauthorized: false
})
});

function sleep(delay: number) {
return new Promise(reslove => {
setTimeout(reslove, delay)
})
}
import { CrawelerDomains } from "../const";

export class BiquCrawler implements Craweler {

private readonly baseURL = "https://www.biqugee6.com";
private readonly baseURL = <string>CrawelerDomains.get("biquURL");
private readonly defaultEncode = "utf-8";
private readonly requestHeaders = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9",
};

async searchBook(keyWord: string): Promise<Map<string, string>> {
let data: string = "";
let self = this;
let count = 0;
const retryCount = 5;
let result;
while (count < retryCount) {
try {
const response = await ignoreSSL.get(self.baseURL + "/search.php", {
params: { keyword: keyWord }
});
result = response;
if (response.data.indexOf("Verify") !== -1) {
count++;
await sleep(1000);
continue;
}
data = response.data;
break;
} catch (error: any) {
window.showErrorMessage(error.message);
throw error;
}
}

if (count >= retryCount) {
let error_msg = "遭遇验证码次数过多，稍后再试吧";
window.showErrorMessage(error_msg);
throw new Error(error_msg);
let data: string;
try {
const response = await axios.get(this.baseURL + "/search/", {
headers: this.requestHeaders,
params: { keyword: keyWord },
responseType: "arraybuffer",
});
data = iconv.decode(response.data, this.defaultEncode);
} catch (error: any) {
window.showErrorMessage(error.message);
throw error;
}

const $ = cheerioModule.load(data);
let choices = new Map<string, string>();
$("a.result-game-item-title-link").each(function (_i, ele) {
choices.set($(ele).prop("title"), self.baseURL + $(ele).prop("href"));
$("a[href^='/']").each((_i, ele) => {
const title = $(ele).text().trim();
const href = $(ele).prop("href");
if (!title || !href || !/^\/\d+$/.test(href)) {
return;
}
if (!choices.has(title)) {
choices.set(title, this.baseURL + href);
}
});
if (choices.size == 0) {
console.log(result)
}
return choices;

}

async findChapterURL(url: string): Promise<Map<string, string>> {
let data: string;
let self = this;
try {
const response = await axios.get(url, {responseType: "arraybuffer"});
const response = await axios.get(url, {
headers: this.requestHeaders,
responseType: "arraybuffer",
});
data = iconv.decode(response.data, this.defaultEncode);
} catch (error: any) {
window.showErrorMessage(error.message);
Expand All @@ -78,9 +58,16 @@ export class BiquCrawler implements Craweler {

const $ = cheerioModule.load(data);
let choices = new Map<string, string>();
$("#list a").each(function (_i, ele) {
choices.set($(ele).text(), self.baseURL + $(ele).prop("href"));
$("a[href^='/']").each((_i, ele) => {
const title = $(ele).text().trim();
const href = $(ele).prop("href");
if (!title || !href || !title.startsWith("第")) {
return;
}
if (!choices.has(title)) {
choices.set(title, this.baseURL + href);
}
});
return choices;
}
}
}
114 changes: 93 additions & 21 deletions src/crawler/caimo.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,51 +3,123 @@ import axios from "axios";
import iconv = require('iconv-lite');
import { window } from "vscode";
import { Craweler } from "./interface";
import { CrawelerDomains } from "../const";

const querystring = require('querystring');

type BookSearchResult = Map<string, string>;

export class CaimoCrawler implements Craweler {

private readonly baseURL = "https://www.caimoge.net";
private readonly baseURL = <string>CrawelerDomains.get("caimoURL");
private readonly defaultEncode = "utf-8";
private readonly requestHeaders = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
"Accept-Language": "zh-CN,zh;q=0.9",
};

private async fetchHTML(url: string, params?: Record<string, string>): Promise<string> {
const response = await axios.get(url, {
headers: this.requestHeaders,
params,
responseType: "arraybuffer",
});
return iconv.decode(response.data, this.defaultEncode);
}

private parseBookLinks(data: string): BookSearchResult {
const $ = cheerioModule.load(data);
const choices = new Map<string, string>();
$("a[href^='/txt/']").each((_i, ele) => {
const title = $(ele).text().trim();
const href = $(ele).prop("href");
if (!title || !href) {
return;
}
if (!choices.has(title)) {
choices.set(title, this.baseURL + href);
}
});
return choices;
}

private async searchFromShelf(keyWord: string): Promise<BookSearchResult> {
const seedPages = [
"/",
"/shuku/",
"/rank/",
"/rank/weekvisit/",
"/rank/monthvisit/",
"/rank/goodnum/",
"/shuku/xuanhuan/1.html",
"/shuku/wuxia/1.html",
"/shuku/yanqing/1.html",
"/shuku/dushi/1.html",
"/shuku/lishi/1.html",
"/shuku/youxi/1.html",
"/shuku/kehuan/1.html",
"/shuku/tongren/1.html",
"/shuku/qita/1.html",
];
const normalizedKeyword = keyWord.trim().toLowerCase();
const matches = new Map<string, string>();

for (const page of seedPages) {
const data = await this.fetchHTML(this.baseURL + page);
const choices = this.parseBookLinks(data);
for (const [title, url] of choices) {
if (title.toLowerCase().includes(normalizedKeyword)) {
matches.set(title, url);
}
}
}
return matches;
}

async searchBook(keyWord: string): Promise<Map<string, string>> {
let data: string;
let self = this;
try {
const response = await axios.post(self.baseURL + "/search/", querystring.stringify({ searchkey: keyWord }));
data = response.data;
const data = await this.fetchHTML(this.baseURL + "/search/", { searchkey: keyWord });
const choices = this.parseBookLinks(data);
if (choices.size > 0) {
return choices;
}
} catch (error: any) {
if (error?.response?.status && error.response.status !== 404) {
window.showWarningMessage(`站内搜索不可用，改用书库检索: ${error.message}`);
}
}

try {
const choices = await this.searchFromShelf(keyWord);
if (choices.size === 0) {
window.showWarningMessage("当前书源未找到匹配书籍，可尝试更完整的书名");
}
return choices;
} catch (error: any) {
window.showErrorMessage(error.message);
throw error;
}

const $ = cheerioModule.load(data);
let choices = new Map<string, string>();
$("#sitembox h3>a").each(function (_i, ele) {
choices.set($(ele).text(), self.baseURL + $(ele).prop("href"));
});
return choices;

}

async findChapterURL(url: string): Promise<Map<string, string>> {
let data: string;
let self = this;
try {
const response = await axios.get(url, {responseType: "arraybuffer"});
data = iconv.decode(response.data, this.defaultEncode);
data = await this.fetchHTML(url);
} catch (error: any) {
window.showErrorMessage(error.message);
throw error;
}

const $ = cheerioModule.load(data);
let choices = new Map<string, string>();
$("#readerlist ul a").each(function (_i, ele) {
choices.set($(ele).text(), self.baseURL + $(ele).prop("href"));
$("a[href^='/read/']").each((_i, ele) => {
const title = $(ele).text().trim();
const href = $(ele).prop("href");
if (!title || !href || !title.startsWith("第")) {
return;
}
if (!choices.has(title)) {
choices.set(title, this.baseURL + href);
}
});
return choices;
}
}
}
Loading