diff --git a/docs/community_growth_20k.md b/docs/community_growth_20k.md index be6229cc0..e2993c618 100644 --- a/docs/community_growth_20k.md +++ b/docs/community_growth_20k.md @@ -371,7 +371,7 @@ Paste the Markdown output into launch notes, weekly community updates, or releas - [x] Pin a GitHub discussion for the release. On 2026-07-23, [`modelscope/FunASR#3376`](https://github.com/modelscope/FunASR/discussions/3376) was pinned through the repository's authenticated Discussions admin UI. GitHub returned `Discussion has successfully been pinned`, and the admin controls now expose `Edit pinned discussion` and `Unpin discussion`; the `v1.3.26` OpenAI-compatible ASR, vLLM, and llama.cpp release story is therefore visible at the top of the public Discussions page. - [x] Triage all new issues within 48 hours. On 2026-07-23, all four repositories were patrolled with the GitHub token on `ind-gpu8`: `modelscope/FunASR`, `FunAudioLLM/Fun-ASR`, `FunAudioLLM/SenseVoice`, and `modelscope/FunClip` all had `0` open pull requests; Fun-ASR, SenseVoice, and FunClip had `0` open issues. The only open FunASR issues were already triaged: #3302 is labeled `question` + `needs feedback` and is waiting for the reporter to retest the merged hotword/postprocess fix, while #3104 is labeled `enhancement` + `needs maintainer decision` and is waiting on release-scope confirmation for RL training code. No unlabelled or unowned open issue remained in the 48-hour window. - [x] Convert top 3 support questions into docs. On 2026-07-23, the latest 80 FunASR issues were triaged into three repeat support blockers: install / hub / model-id selection (#3321, #3045, #3042, #2973, #2976), llama.cpp / GGUF / CUDA-Vulkan runtime packages (#3298, #3297, #3296, #3289, #3243), and realtime / VAD / vLLM / server output behavior (#3101, #3109, #3038, #3031, #2968, #2965). `docs/troubleshooting.md` and `docs/troubleshooting_zh.md` now start with a Top support questions section that maps those three issue clusters to the existing install, model-download, OpenAI-compatible server, WebSocket, and llama.cpp / GGUF FAQ anchors. -- [ ] Track stars, PyPI downloads, issue volume, and docs traffic. `collect_growth_metrics.py` now tracks GitHub stars, open issues, and open pull requests for the four core repositories and adds PyPIStats download windows for `funasr` as `downloads_last_7_days` and `downloads_last_30_days`. On 2026-07-23, the live PyPIStats window was refreshed at `116,407` downloads in 7 days and `442,556` downloads in 30 days through `2026-07-21`; successful runs now write `/cpfs_speech/user/zhifu.gzf/.cache/funasr-ops/pypi-downloads-funasr.json`, and failed live runs fall back to that cache with status `stale_available` instead of dropping the download metric during SSL/429 outages. GitHub/PyPI tracking is now covered; docs traffic remains unavailable until an analytics credential or export is connected. +- [x] Track stars, PyPI downloads, issue volume, and docs traffic. `collect_growth_metrics.py` tracks GitHub stars, open issues, and open pull requests for the four core repositories and adds PyPIStats download windows for `funasr` as `downloads_last_7_days` and `downloads_last_30_days`. On 2026-07-23, the live PyPIStats window was refreshed at `117,850` downloads in 7 days and `445,755` downloads in 30 days through `2026-07-22`; successful runs write `/cpfs_speech/user/zhifu.gzf/.cache/funasr-ops/pypi-downloads-funasr.json`, and failed live runs fall back to that cache with status `stale_available` instead of dropping the metric during SSL/429 outages. `scripts/collect_website_traffic.py` completes the docs-traffic side from plain or gzip Nginx logs with `python scripts/collect_website_traffic.py --log '/var/log/nginx/access.log*' --days 30 --format json`; it filters bots, static assets, unsuccessful responses, and out-of-window requests, and emits aggregate counts only so raw client IPs and user agents are never emitted. A production run on `funasr-web` over 15 rotated logs at `2026-07-23T04:52:08Z` measured 30-day traffic at `88,239` page views / `47,084` approximate unique visitors, including docs at `15,501` / `11,843` and blog at `27,901` / `22,716`. The matching 7-day run measured `21,915` / `12,668`, including docs at `4,638` / `3,612` and blog at `6,901` / `5,590`. Stars, PyPI downloads, issue volume, and owned-site traffic now have repeatable baselines for patrols. ## Release-note template diff --git a/scripts/collect_website_traffic.py b/scripts/collect_website_traffic.py new file mode 100644 index 000000000..f64dcd550 --- /dev/null +++ b/scripts/collect_website_traffic.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Aggregate privacy-safe funasr.com traffic metrics from Nginx access logs.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import glob +import gzip +import hashlib +import json +import os +import re +from collections import Counter, defaultdict +from pathlib import Path +from typing import Iterable, Iterator, NamedTuple, Optional, Sequence +from urllib.parse import unquote, urlsplit + + +LOG_PATTERN = re.compile( + r'^(?P\S+) \S+ \S+ \[(?P[^]]+)] ' + r'"(?P[A-Z]+) (?P\S+) [^"]+" ' + r'(?P\d{3}) \S+ "(?P[^"]*)" "(?P[^"]*)"$' +) +BOT_PATTERN = re.compile( + r"bot|spider|crawler|slurp|headless|lighthouse|pagespeed|curl/|wget/|" + r"python-requests|aiohttp|scrapy|go-http-client|uptime|monitoring|zgrab|masscan", + re.IGNORECASE, +) +STATIC_EXTENSIONS = { + ".avif", + ".css", + ".eot", + ".gif", + ".ico", + ".jpeg", + ".jpg", + ".js", + ".json", + ".map", + ".mp3", + ".mp4", + ".ogg", + ".pdf", + ".png", + ".svg", + ".txt", + ".wav", + ".webm", + ".webp", + ".woff", + ".woff2", + ".xml", + ".zip", +} +IGNORED_PREFIXES = ("/.well-known/", "/assets/", "/fonts/", "/images/", "/static/") +DOC_PREFIXES = ("/docs/", "/en/docs/", "/guide/", "/en/guide/") +DOC_FILENAMES = { + "agent.html", + "benchmark.html", + "cli-tutorial.html", + "download.html", + "ecosystem.html", + "llama-cpp.html", + "models.html", + "quickstart.html", + "vs-whisper.html", +} +SECTIONS = ("homepage", "docs", "blog", "donors", "other") + + +class AccessRecord(NamedTuple): + client_ip: str + timestamp: dt.datetime + method: str + path: str + status: int + referrer_host: str + user_agent: str + + +def normalize_path(target: str) -> str: + path = unquote(urlsplit(target).path or "/") + if not path.startswith("/"): + path = f"/{path}" + return path + + +def referrer_host(referrer: str) -> str: + if not referrer or referrer == "-": + return "" + return (urlsplit(referrer).hostname or "").lower() + + +def parse_access_log_line(line: str) -> Optional[AccessRecord]: + match = LOG_PATTERN.match(line.rstrip("\n")) + if match is None: + return None + try: + timestamp = dt.datetime.strptime(match.group("timestamp"), "%d/%b/%Y:%H:%M:%S %z") + status = int(match.group("status")) + except ValueError: + return None + return AccessRecord( + client_ip=match.group("client_ip"), + timestamp=timestamp, + method=match.group("method"), + path=normalize_path(match.group("target")), + status=status, + referrer_host=referrer_host(match.group("referrer")), + user_agent=match.group("user_agent"), + ) + + +def is_page_path(path: str) -> bool: + lowered = path.lower() + if lowered in {"/favicon.ico", "/robots.txt", "/sitemap.xml"}: + return False + if lowered.startswith(IGNORED_PREFIXES): + return False + return Path(lowered).suffix not in STATIC_EXTENSIONS + + +def classify_page(path: str) -> str: + lowered = path.lower() + if lowered in {"/", "/index.html", "/en", "/en/", "/en/index.html"}: + return "homepage" + if lowered.startswith(("/blog/", "/en/blog/")): + return "blog" + if lowered in {"/donors.html", "/en/donors.html"}: + return "donors" + if lowered.startswith(DOC_PREFIXES) or Path(lowered).name in DOC_FILENAMES: + return "docs" + return "other" + + +def anonymized_visitor(record: AccessRecord, salt: bytes) -> bytes: + value = f"{record.client_ip}\0{record.user_agent}".encode("utf-8", errors="replace") + return hashlib.blake2b(value, key=salt, digest_size=16).digest() + + +def aggregate_records( + records: Iterable[Optional[AccessRecord]], + *, + now: dt.datetime, + days: int, + top_limit: int, + visitor_salt: Optional[bytes] = None, +) -> dict: + if now.tzinfo is None: + raise ValueError("now must include a timezone") + if days <= 0: + raise ValueError("days must be positive") + if top_limit <= 0: + raise ValueError("top_limit must be positive") + + cutoff = now - dt.timedelta(days=days) + salt = visitor_salt or os.urandom(16) + page_views = 0 + visitors = set() + section_views = Counter() + section_visitors = defaultdict(set) + path_views = Counter() + path_visitors = defaultdict(set) + referrer_views = Counter() + filtered = Counter() + + for record in records: + if record is None: + filtered["malformed"] += 1 + continue + record_time = record.timestamp.astimezone(now.tzinfo) + if record_time < cutoff or record_time > now: + filtered["outside_window"] += 1 + continue + if record.method not in {"GET", "HEAD"}: + filtered["non_page_methods"] += 1 + continue + if record.status != 200: + filtered["non_success"] += 1 + continue + if BOT_PATTERN.search(record.user_agent): + filtered["bots"] += 1 + continue + if not is_page_path(record.path): + filtered["static_assets"] += 1 + continue + + visitor = anonymized_visitor(record, salt) + section = classify_page(record.path) + page_views += 1 + visitors.add(visitor) + section_views[section] += 1 + section_visitors[section].add(visitor) + path_views[record.path] += 1 + path_visitors[record.path].add(visitor) + if record.referrer_host and not ( + record.referrer_host == "funasr.com" or record.referrer_host.endswith(".funasr.com") + ): + referrer_views[record.referrer_host] += 1 + + sections = { + section: { + "page_views": section_views[section], + "unique_visitors": len(section_visitors[section]), + } + for section in SECTIONS + } + top_paths = [ + { + "path": path, + "page_views": views, + "unique_visitors": len(path_visitors[path]), + } + for path, views in sorted(path_views.items(), key=lambda item: (-item[1], item[0]))[:top_limit] + ] + top_referrers = [ + {"host": host, "page_views": views} + for host, views in sorted(referrer_views.items(), key=lambda item: (-item[1], item[0]))[:top_limit] + ] + filter_names = ( + "malformed", + "outside_window", + "non_page_methods", + "non_success", + "bots", + "static_assets", + ) + return { + "collected_at_utc": now.astimezone(dt.timezone.utc).isoformat(), + "window_start_utc": cutoff.astimezone(dt.timezone.utc).isoformat(), + "days": days, + "page_views": page_views, + "unique_visitors": len(visitors), + "sections": sections, + "top_paths": top_paths, + "top_referrers": top_referrers, + "filtered": {name: filtered[name] for name in filter_names}, + "privacy": "Aggregate counts only; raw client IPs and user agents are never emitted.", + } + + +def iter_log_lines(paths: Iterable[Path]) -> Iterator[str]: + for path in paths: + opener = gzip.open if path.suffix == ".gz" else open + with opener(path, "rt", encoding="utf-8", errors="replace") as handle: + yield from handle + + +def resolve_log_paths(patterns: Sequence[str]) -> list[Path]: + paths = [] + seen = set() + for pattern in patterns: + for match in sorted(glob.glob(pattern)): + path = Path(match) + if path.is_file() and path not in seen: + paths.append(path) + seen.add(path) + return paths + + +def collect_traffic( + paths: Sequence[Path], + *, + now: dt.datetime, + days: int, + top_limit: int, +) -> dict: + records = (parse_access_log_line(line) for line in iter_log_lines(paths)) + result = aggregate_records(records, now=now, days=days, top_limit=top_limit) + result["source_file_count"] = len(paths) + return result + + +def parse_now(value: Optional[str]) -> dt.datetime: + if value is None: + return dt.datetime.now(dt.timezone.utc) + parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise argparse.ArgumentTypeError("--now must include a timezone") + return parsed + + +def render_text(metrics: dict) -> str: + lines = [ + f"funasr.com traffic ({metrics['days']} days)", + f"Page views: {metrics['page_views']:,}", + f"Approximate unique visitors: {metrics['unique_visitors']:,}", + ] + for section in SECTIONS: + values = metrics["sections"][section] + lines.append( + f"{section}: {values['page_views']:,} views / " + f"{values['unique_visitors']:,} approximate visitors" + ) + return "\n".join(lines) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--log", + action="append", + required=True, + help="Access-log path or glob. Repeat for multiple files.", + ) + parser.add_argument("--days", type=int, default=30, help="Trailing window length (default: 30).") + parser.add_argument("--top", type=int, default=10, help="Number of top paths/referrers (default: 10).") + parser.add_argument("--now", help="Timezone-aware ISO timestamp for reproducible backfills.") + parser.add_argument("--format", choices=("text", "json"), default="text") + args = parser.parse_args(argv) + + try: + now = parse_now(args.now) + paths = resolve_log_paths(args.log) + if not paths: + parser.error("no access-log files matched --log") + metrics = collect_traffic(paths, now=now, days=args.days, top_limit=args.top) + except (OSError, ValueError) as error: + parser.error(str(error)) + + if args.format == "json": + print(json.dumps(metrics, ensure_ascii=False, indent=2)) + else: + print(render_text(metrics)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_collect_website_traffic.py b/tests/test_collect_website_traffic.py new file mode 100644 index 000000000..5f9593ab5 --- /dev/null +++ b/tests/test_collect_website_traffic.py @@ -0,0 +1,161 @@ +import gzip +import importlib.util +import json +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def load_module(): + script = Path(__file__).resolve().parents[1] / "scripts" / "collect_website_traffic.py" + spec = importlib.util.spec_from_file_location("collect_website_traffic", script) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def access_line( + ip, + timestamp, + path, + *, + status=200, + referrer="-", + user_agent="Mozilla/5.0 Chrome/142.0", +): + return ( + f'{ip} - - [{timestamp}] "GET {path} HTTP/2.0" {status} 123 ' + f'"{referrer}" "{user_agent}"\n' + ) + + +def test_parse_access_log_line_normalizes_path_and_referrer(): + module = load_module() + record = module.parse_access_log_line( + access_line( + "203.0.113.5", + "23/Jul/2026:11:00:00 +0800", + "/docs/guide.html?source=release", + referrer="https://www.google.com/search?q=funasr", + ) + ) + + assert record.client_ip == "203.0.113.5" + assert record.timestamp.isoformat() == "2026-07-23T11:00:00+08:00" + assert record.method == "GET" + assert record.path == "/docs/guide.html" + assert record.status == 200 + assert record.referrer_host == "www.google.com" + + +def test_aggregate_filters_bots_assets_failures_and_old_requests(): + module = load_module() + lines = [ + access_line("203.0.113.10", "23/Jul/2026:09:00:00 +0800", "/"), + access_line("203.0.113.10", "23/Jul/2026:09:01:00 +0800", "/docs/quickstart.html"), + access_line( + "203.0.113.20", + "23/Jul/2026:09:02:00 +0800", + "/en/blog/local-asr.html", + referrer="https://news.ycombinator.com/item?id=1", + ), + access_line("203.0.113.20", "23/Jul/2026:09:03:00 +0800", "/donors.html"), + access_line( + "198.51.100.1", + "23/Jul/2026:09:04:00 +0800", + "/docs/quickstart.html", + user_agent="Googlebot/2.1", + ), + access_line("198.51.100.2", "23/Jul/2026:09:05:00 +0800", "/assets/app.js"), + access_line("198.51.100.3", "23/Jul/2026:09:06:00 +0800", "/docs/missing.html", status=404), + access_line("198.51.100.4", "20/Jun/2026:09:07:00 +0800", "/docs/old.html"), + ] + records = [module.parse_access_log_line(line) for line in lines] + + metrics = module.aggregate_records( + records, + now=datetime(2026, 7, 23, 12, 0, tzinfo=timezone.utc), + days=30, + top_limit=10, + ) + + assert metrics["page_views"] == 4 + assert metrics["unique_visitors"] == 2 + assert metrics["sections"] == { + "homepage": {"page_views": 1, "unique_visitors": 1}, + "docs": {"page_views": 1, "unique_visitors": 1}, + "blog": {"page_views": 1, "unique_visitors": 1}, + "donors": {"page_views": 1, "unique_visitors": 1}, + "other": {"page_views": 0, "unique_visitors": 0}, + } + assert metrics["filtered"]["bots"] == 1 + assert metrics["filtered"]["static_assets"] == 1 + assert metrics["filtered"]["non_success"] == 1 + assert metrics["filtered"]["outside_window"] == 1 + assert metrics["top_referrers"] == [{"host": "news.ycombinator.com", "page_views": 1}] + serialized = json.dumps(metrics) + assert "203.0.113" not in serialized + assert "Mozilla" not in serialized + + +def test_iter_log_lines_supports_plain_and_gzip_files(tmp_path): + module = load_module() + plain = tmp_path / "access.log" + compressed = tmp_path / "access.log.1.gz" + plain.write_text("plain\n") + with gzip.open(compressed, "wt") as handle: + handle.write("compressed\n") + + assert list(module.iter_log_lines([plain, compressed])) == ["plain\n", "compressed\n"] + + +def test_classify_owned_product_guides_as_docs(): + module = load_module() + + for path in ( + "/quickstart.html", + "/en/quickstart.html", + "/models.html", + "/en/models.html", + "/vs-whisper.html", + "/en/vs-whisper.html", + "/ecosystem.html", + "/en/llama-cpp.html", + ): + assert module.classify_page(path) == "docs" + + +def test_cli_outputs_aggregate_json_without_identifiers(tmp_path): + script = Path(__file__).resolve().parents[1] / "scripts" / "collect_website_traffic.py" + log = tmp_path / "access.log" + log.write_text( + access_line( + "203.0.113.55", + "23/Jul/2026:09:00:00 +0800", + "/docs/deployment.html", + ) + ) + + result = subprocess.run( + [ + sys.executable, + str(script), + "--log", + str(log), + "--now", + "2026-07-23T12:00:00+00:00", + "--days", + "30", + "--format", + "json", + ], + check=True, + capture_output=True, + text=True, + ) + payload = json.loads(result.stdout) + + assert payload["page_views"] == 1 + assert payload["sections"]["docs"]["page_views"] == 1 + assert "203.0.113.55" not in result.stdout diff --git a/tests/test_growth_plan_doc.py b/tests/test_growth_plan_doc.py index 7113b0074..c6cd49bb1 100644 --- a/tests/test_growth_plan_doc.py +++ b/tests/test_growth_plan_doc.py @@ -363,23 +363,36 @@ def test_growth_plan_records_homepage_entrypoint_completion(): assert "[ ] Update homepage hero and docs entry points" not in text -def test_growth_plan_records_metrics_tracking_progress(): +def test_growth_plan_records_metrics_tracking_completion(): text = PLAN.read_text() required_markers = [ - "[ ] Track stars, PyPI downloads, issue volume, and docs traffic.", + "[x] Track stars, PyPI downloads, issue volume, and docs traffic.", "PyPIStats", "`downloads_last_7_days`", "`downloads_last_30_days`", "`collect_growth_metrics.py`", "GitHub stars, open issues, and open pull requests", - "`116,407` downloads in 7 days", - "`442,556` downloads in 30 days", - "`2026-07-21`", + "`117,850` downloads in 7 days", + "`445,755` downloads in 30 days", + "`2026-07-22`", "pypi-downloads-funasr.json", "`stale_available`", "SSL/429 outages", - "docs traffic remains unavailable until an analytics credential or export is connected", + "`scripts/collect_website_traffic.py`", + "`python scripts/collect_website_traffic.py --log '/var/log/nginx/access.log*' --days 30 --format json`", + "plain or gzip Nginx logs", + "raw client IPs and user agents are never emitted", + "over 15 rotated logs", + "`88,239` page views / `47,084` approximate unique visitors", + "docs at `15,501` / `11,843`", + "blog at `27,901` / `22,716`", + "`21,915` / `12,668`", + "docs at `4,638` / `3,612`", + "blog at `6,901` / `5,590`", ] for marker in required_markers: assert marker in text + + assert "[ ] Track stars, PyPI downloads, issue volume, and docs traffic." not in text + assert "docs traffic remains unavailable" not in text