fix(msk-collector): --resume берёт base_url из сохранённого плана
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI / changes (pull_request) Successful in 10s
CI Trade-In / backend-tests (pull_request) Has been skipped
CI Trade-In / browser-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI / changes (pull_request) Successful in 10s
CI Trade-In / backend-tests (pull_request) Has been skipped
CI Trade-In / browser-tests (pull_request) Has been skipped
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
Коридоры считаются под конкретный URL выдачи. При --resume код брал свежий args.base_url, поэтому запуск без повтора --base-url молча качал другую выдачу под тем же batch_id. Теперь URL из плана, расхождение с аргументом — явная ошибка вместо тихого выбора одного из двух.
This commit is contained in:
parent
7b33ed9a45
commit
bde38cdce7
1 changed files with 15 additions and 4 deletions
|
|
@ -527,13 +527,24 @@ async def collect(args: argparse.Namespace) -> int:
|
|||
print(f"Resume по {plan_path}: коридоров {len(plan.corridors)}, "
|
||||
f"готово {done}", flush=True)
|
||||
|
||||
# Resume: URL берём из сохранённого плана, а не из CLI — коридоры посчитаны
|
||||
# именно под него. Расхождение = молчаливая заливка чужой выдачи под тем же
|
||||
# batch_id, поэтому это ошибка, а не тихий приоритет одного из двух.
|
||||
if plan is not None and plan.base_url != args.base_url:
|
||||
raise SystemExit(
|
||||
"--resume: план построен для другого URL."
|
||||
f" В плане {plan.base_url}, в аргументах {args.base_url}."
|
||||
" Убери --base-url (возьмётся из плана) либо начни новый batch_id."
|
||||
)
|
||||
base_url = plan.base_url if plan is not None else args.base_url
|
||||
|
||||
page_budget = None if args.full else args.measure
|
||||
mode = "FULL" if args.full else f"MEASURE<={page_budget}"
|
||||
print(f"Режим: {mode}; batch_id={args.batch_id}; delay={args.delay}s; "
|
||||
f"target={args.target_count}; dry_run={args.dry_run}", flush=True)
|
||||
|
||||
sink = Sink(
|
||||
batch_id=args.batch_id, started_at=started_at, query=args.base_url,
|
||||
batch_id=args.batch_id, started_at=started_at, query=base_url,
|
||||
batch_size=args.batch_size, dry_run=args.dry_run, csv_path=csv_path,
|
||||
ssh_host=args.ssh_host, container=args.container,
|
||||
db_user=args.db_user, db_name=args.db_name,
|
||||
|
|
@ -548,9 +559,9 @@ async def collect(args: argparse.Namespace) -> int:
|
|||
try:
|
||||
if plan is None:
|
||||
print("Строю план коридоров...", flush=True)
|
||||
corridors = await build_plan(loader, scraper, args.base_url,
|
||||
corridors = await build_plan(loader, scraper, base_url,
|
||||
args.target_count, cache)
|
||||
plan = Plan(base_url=args.base_url, target=args.target_count,
|
||||
plan = Plan(base_url=base_url, target=args.target_count,
|
||||
batch_id=args.batch_id, corridors=corridors,
|
||||
created_at=started_at)
|
||||
plan.save(plan_path)
|
||||
|
|
@ -568,7 +579,7 @@ async def collect(args: argparse.Namespace) -> int:
|
|||
if page == 1 and key in cache:
|
||||
lots = cache.pop(key) # страница 1 уже скачана при планировании
|
||||
else:
|
||||
url = build_url(args.base_url, page, corridor.lo, corridor.hi)
|
||||
url = build_url(base_url, page, corridor.lo, corridor.hi)
|
||||
html, _ = await loader.fetch(url)
|
||||
_, lots = parse_page(scraper, html, url)
|
||||
for lot in lots:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue