Add concurrent geocoding with ThreadPoolExecutor
- Use --workers to set concurrent thread count (default 3) - Speed up coordinate lookup by ~3x with 3 workers - Maintain Nominatim rate limiting per thread Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -17,6 +17,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import concurrent.futures
|
||||||
import csv
|
import csv
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
@@ -198,10 +199,15 @@ def geocode(address: str, station_name: str, city: str, district: str) -> Option
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def add_coordinates(stations: List[Dict[str, str]], delay: float = 1.0) -> List[Dict[str, str]]:
|
def add_coordinates(stations: List[Dict[str, str]], delay: float = 1.0, workers: int = 3) -> List[Dict[str, str]]:
|
||||||
"""為站點資料加上座標資訊(使用快取)。"""
|
"""為站點資料加上座標資訊(使用快取+並發查詢)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
stations: 站點列表
|
||||||
|
delay: 每個查詢間隔秒數(Nominatim 限制約 1 秒)
|
||||||
|
workers: 並發查詢執行緒數(預設 3,Nominatim 免費版限制)
|
||||||
|
"""
|
||||||
cache = load_coords_cache()
|
cache = load_coords_cache()
|
||||||
cache_updated = False
|
|
||||||
missing_coords = []
|
missing_coords = []
|
||||||
|
|
||||||
# 先從快取讀取
|
# 先從快取讀取
|
||||||
@@ -215,10 +221,11 @@ def add_coordinates(stations: List[Dict[str, str]], delay: float = 1.0) -> List[
|
|||||||
station["經度"] = ""
|
station["經度"] = ""
|
||||||
missing_coords.append(station)
|
missing_coords.append(station)
|
||||||
|
|
||||||
# 查詢缺少的座標
|
# 並發查詢缺少的座標
|
||||||
if missing_coords:
|
if missing_coords:
|
||||||
print(f"快取命中: {len(stations) - len(missing_coords)}/{len(stations)},需查詢: {len(missing_coords)}", file=sys.stderr)
|
print(f"快取命中: {len(stations) - len(missing_coords)}/{len(stations)},需查詢: {len(missing_coords)}", file=sys.stderr)
|
||||||
for station in missing_coords:
|
|
||||||
|
def query_one(station: Dict[str, str]) -> tuple[Dict[str, str], Optional[Dict[str, float]]]:
|
||||||
full_address = f"{station['縣市']}{station['鄉鎮區']}{station['地址']}"
|
full_address = f"{station['縣市']}{station['鄉鎮區']}{station['地址']}"
|
||||||
coords = geocode(
|
coords = geocode(
|
||||||
address=full_address,
|
address=full_address,
|
||||||
@@ -226,15 +233,19 @@ def add_coordinates(stations: List[Dict[str, str]], delay: float = 1.0) -> List[
|
|||||||
city=station["縣市"],
|
city=station["縣市"],
|
||||||
district=station["鄉鎮區"],
|
district=station["鄉鎮區"],
|
||||||
)
|
)
|
||||||
|
time.sleep(delay) # 避免 Nominatim 請求過快
|
||||||
|
return (station, coords)
|
||||||
|
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
|
||||||
|
futures = {executor.submit(query_one, s): s for s in missing_coords}
|
||||||
|
for future in concurrent.futures.as_completed(futures):
|
||||||
|
station, coords = future.result()
|
||||||
if coords:
|
if coords:
|
||||||
station["緯度"] = str(coords["lat"])
|
station["緯度"] = str(coords["lat"])
|
||||||
station["經度"] = str(coords["lon"])
|
station["經度"] = str(coords["lon"])
|
||||||
key = make_cache_key(station)
|
key = make_cache_key(station)
|
||||||
cache[key] = coords
|
cache[key] = coords
|
||||||
cache_updated = True
|
|
||||||
time.sleep(delay) # 避免 Nominatim 請求過快
|
|
||||||
|
|
||||||
if cache_updated:
|
|
||||||
save_coords_cache(cache)
|
save_coords_cache(cache)
|
||||||
|
|
||||||
return stations
|
return stations
|
||||||
@@ -365,7 +376,8 @@ def main() -> None:
|
|||||||
parser.add_argument("--format", choices=["text", "json", "csv"], default="text")
|
parser.add_argument("--format", choices=["text", "json", "csv"], default="text")
|
||||||
parser.add_argument("--out", help="輸出到檔案路徑")
|
parser.add_argument("--out", help="輸出到檔案路徑")
|
||||||
parser.add_argument("--coords", action="store_true", help="加上座標資訊(使用 Nominatim OSM API)")
|
parser.add_argument("--coords", action="store_true", help="加上座標資訊(使用 Nominatim OSM API)")
|
||||||
parser.add_argument("--delay", type=float, default=1.0, help="座標查詢間隔秒數(預設 1.0,Nominatim 限制)")
|
parser.add_argument("--delay", type=float, default=1.0, help="座標查詢間隔秒數(預設 1.0)")
|
||||||
|
parser.add_argument("--workers", type=int, default=3, help="並發查詢執行緒數(預設 3)")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
html = build_query_html(city=args.city, keyword=args.keyword)
|
html = build_query_html(city=args.city, keyword=args.keyword)
|
||||||
@@ -373,7 +385,7 @@ def main() -> None:
|
|||||||
stations = filter_stations(stations, args.keyword)
|
stations = filter_stations(stations, args.keyword)
|
||||||
|
|
||||||
if args.coords:
|
if args.coords:
|
||||||
stations = add_coordinates(stations, delay=args.delay)
|
stations = add_coordinates(stations, delay=args.delay, workers=args.workers)
|
||||||
|
|
||||||
if args.format == "json":
|
if args.format == "json":
|
||||||
output_json(stations, args.out)
|
output_json(stations, args.out)
|
||||||
|
|||||||
Reference in New Issue
Block a user