adscrawler/pyproject.toml at main · appgoblin-dev/adscrawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
[project]
authors = [{ name = "James O'Claire" }]
name = "adscrawler"
description = "A monorepo for crawling app stores, app-ads.txt, waydroid, ad creatives, and more."
requires-python = ">= 3.12"
dynamic = ["version"]


dependencies = [
    'requests',
    'pandas',
    'numpy',
    'sqlalchemy',
    'google-play-scraper @ git+https://github.com/ddxv/google-play-scraper',
    'tldextract',
    'itunes-app-scraper-dmi @ git+https://github.com/ddxv/itunes-app-scraper.git',
    'asyncssh',
    'selenium',
    'pyyaml',
    "psycopg[binary]",
    "beautifulsoup4",
    "nltk",
    "rake_nltk",
    "langdetect",
    "mitmproxy",
    "cloudscraper",
    "geoip2",
    "boto3",
    "duckdb",
    "imagehash",
    "protod",
    "pyarrow",
]


[project.optional-dependencies]
dev = ["pre-commit", "pytest","scikit-learn", "spacy" ]


[build-system]
requires = ["setuptools>=64", "setuptools_scm>=8"]
build-backend = "setuptools.build_meta"


[tool.setuptools_scm]

[tool.setuptools.packages]
find = {}

[tool.setuptools.package-data]
adscrawler = ["*.js"]

[tool.ruff]
target-version = "py313"

[tool.ruff.lint]
select = [
    "A",   # flake8 builtins
    "E",   # errors
    "W",   # pycodestyle Warnings
    "F",   # PyFlake
    "B",   # flake8-bugbear rules
    "I",   # import sort
    "N",   # PEP8 naming rules
    "UP",  # pyupgrade, warnings for common mistakes for new versions
    "PLW", # pyflake
    "PLC", # pyflake
    "PLE", # pyflake
    "PD",  # pandas-vet
]
ignore = [
    "E501",    # line length
    "W291",    # trailing space at end of line
    "ERA001",  # no commented out code
    "D103",    # Docstrings for all functions
    "D102",    # Docstrings for all methods
    "D101",    # Docstrings for all classes
    "G004",    # no f-string inside logging.info
    "D211",    # one-blank-line-before-class, conflict with D203
    "D213",    # multi-line-summary-second-line, conflict with D212
    "PLW2901", # for loop variable overwritten by assignment, falling for pandas rename
    "RET504",  # Unnecessary assignment to variable for returning, I prefer it for readability
    "PD015",   # Use df.merge instead of pd.merge, I prefer the latter for readability
]
unfixable = ["B"]
fixable = ["ALL"]

[tool.mypy]
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true

[tool.sqlfluff.core]
templater = 'placeholder'
verbose = 1
dialect = 'postgres'
large_file_skip_byte_limit = 20000
exclude_rules = [
    'AM04', # using *, agreed this should be TODO,
    'LT08', # Avoid new lines after CTEs to help copy pasting between editors
]
[tool.sqlfluff.rules.layout.long_lines]
ignore_comment_lines = true


[tool.sqlfluff.templater.placeholder]
param_style = 'colon'