diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 2eae65c684..77108acbe1 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -436,7 +436,7 @@ TURNSTILE_SECRET_KEY= # Proxy provider selection. Selects a ProxyProvider implementation registered in -# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse". +# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse", "nodemaven". # PROXY_PROVIDER=custom # Proxy endpoint(s), shared across providers — PROXY_PROVIDER picks the behavior, @@ -447,6 +447,8 @@ TURNSTILE_SECRET_KEY= # provider). For "dataimpulse", country is a "__cr." username suffix the # provider parses for geoip-match, e.g.: # PROXY_URL=http://your_token__cr.us:your_password@gw.dataimpulse.com:823 +# For "nodemaven", country is a "-country-" username parameter, e.g.: +# PROXY_URL=http://your_login-country-us:your_password@gate.nodemaven.com:8080 # PROXY_URLS: a comma-separated pool the "custom" provider rotates client-side # (cyclic); server-side-rotating gateways ignore it. # PROXY_URL=http://user:pass@host:port diff --git a/surfsense_backend/app/utils/proxy/providers/nodemaven.py b/surfsense_backend/app/utils/proxy/providers/nodemaven.py new file mode 100644 index 0000000000..2b7587a55e --- /dev/null +++ b/surfsense_backend/app/utils/proxy/providers/nodemaven.py @@ -0,0 +1,99 @@ +"""NodeMaven residential / rotating proxy provider. + +Takes the shared ``PROXY_URL`` env, like ``custom`` and ``dataimpulse``. What it +adds is NodeMaven's username syntax, ``---...``: + +* :meth:`get_location` reads the ``-country-`` parameter so the crawler's + geoip-match can align the browser locale with the exit country. +* Country routing and sticky sessions rewrite the username with + ``-country-`` and ``-sid-``. + +Rotation happens server-side (a fresh exit IP per connection without ``sid``), +so this is NOT :pyattr:`~ProxyProvider.is_pool_backed`. + +Example URL:: + + http://your_login-country-us:your_password@gate.nodemaven.com:8080 + +``region``, ``city`` and ``isp`` belong to a country, so they are dropped when +the country is changed and kept otherwise; ``ttl`` and other parameters are +kept as written. +""" + +import re +from urllib.parse import quote, unquote, urlsplit, urlunsplit + +from app.config import Config +from app.utils.proxy.base import ProxyProvider + +_COUNTRY_RE = re.compile(r"-country-([A-Za-z_]+)") +_LOCATION_RE = re.compile(r"-(?:country|region|city|isp)-[A-Za-z0-9_]+") +_SESSION_RE = re.compile(r"-sid-[A-Za-z0-9]+") + + +def _safe_session_id(session_id: str) -> str: + # "-" separates parameters in the username, so a session id is alphanumeric. + safe_id = re.sub(r"[^A-Za-z0-9]", "", session_id) + if not safe_id: + raise ValueError("session_id must contain at least one letter or digit") + return safe_id + + +def _safe_country(country: str | None) -> str | None: + if country is None: + return None + safe_country = re.sub(r"[^a-z]", "", country.lower()) + return safe_country or None + + +class NodeMavenProvider(ProxyProvider): + """Provider for a NodeMaven proxy URL in the shared ``PROXY_URL`` env.""" + + name = "nodemaven" + + def get_proxy_url(self) -> str | None: + url = (Config.PROXY_URL or "").strip() + return url or None + + def get_location(self) -> str: + """Country from the ``-country-`` username parameter, or ``""``.""" + url = self.get_proxy_url() + if not url: + return "" + match = _COUNTRY_RE.search(unquote(urlsplit(url).username or "")) + return match.group(1).lower() if match else "" + + def _rewrite_proxy_url( + self, *, country: str | None = None, session_id: str | None = None + ) -> str | None: + url = self.get_proxy_url() + if not url: + return None + parts = urlsplit(url) + username = _SESSION_RE.sub("", unquote(parts.username or "")) + safe_country = _safe_country(country) + if safe_country is not None and safe_country != self.get_location(): + username = _LOCATION_RE.sub("", username) + f"-country-{safe_country}" + if session_id is not None: + username += f"-sid-{_safe_session_id(session_id)}" + userinfo = quote(username, safe="") + if parts.password is not None: + userinfo += f":{quote(unquote(parts.password), safe='')}" + netloc = f"{userinfo}@{parts.hostname or ''}" + if parts.port: + netloc += f":{parts.port}" + return urlunsplit( + (parts.scheme, netloc, parts.path, parts.query, parts.fragment) + ) + + def get_geo_proxy_url(self, country: str | None = None) -> str | None: + """Return the configured URL routed to ``country`` when one is given.""" + if not _safe_country(country): + return self.get_proxy_url() + return self._rewrite_proxy_url(country=country) + + def get_sticky_proxy_url( + self, session_id: str, country: str | None = None + ) -> str | None: + """Return the configured URL with a session id and optional country.""" + return self._rewrite_proxy_url(country=country, session_id=session_id) diff --git a/surfsense_backend/app/utils/proxy/registry.py b/surfsense_backend/app/utils/proxy/registry.py index b222af83c2..4830fef100 100644 --- a/surfsense_backend/app/utils/proxy/registry.py +++ b/surfsense_backend/app/utils/proxy/registry.py @@ -11,6 +11,7 @@ from app.utils.proxy.base import ProxyProvider from app.utils.proxy.providers.custom import CustomProxyProvider from app.utils.proxy.providers.dataimpulse import DataImpulseProvider +from app.utils.proxy.providers.nodemaven import NodeMavenProvider logger = logging.getLogger(__name__) @@ -18,6 +19,7 @@ _PROVIDERS: dict[str, type[ProxyProvider]] = { CustomProxyProvider.name: CustomProxyProvider, DataImpulseProvider.name: DataImpulseProvider, + NodeMavenProvider.name: NodeMavenProvider, } # BYO ``custom`` is the neutral default: it needs no vendor and returns no proxy diff --git a/surfsense_backend/tests/unit/utils/proxy/test_nodemaven_provider.py b/surfsense_backend/tests/unit/utils/proxy/test_nodemaven_provider.py new file mode 100644 index 0000000000..b7500ad618 --- /dev/null +++ b/surfsense_backend/tests/unit/utils/proxy/test_nodemaven_provider.py @@ -0,0 +1,160 @@ +"""Unit tests for the NodeMavenProvider. + +Takes a single full URL (like ``custom``); the vendor-specific bit is the +``-country-`` / ``-sid-`` username syntax used for geoip-match, country +routing and sticky sessions. +""" + +import pytest + +from app.config import Config +from app.utils.proxy.providers.nodemaven import NodeMavenProvider + +pytestmark = pytest.mark.unit + +_URL = "http://acct-country-us:secret@gate.nodemaven.com:8080" + + +@pytest.fixture(autouse=True) +def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", None) + + +def test_returns_configured_url(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + provider = NodeMavenProvider() + + assert provider.is_pool_backed is False + assert provider.get_proxy_url() == _URL + assert provider.get_requests_proxies() == {"http": _URL, "https": _URL} + + +def test_location_parsed_from_country_param(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + assert NodeMavenProvider().get_location() == "us" + + +def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + Config, + "PROXY_URL", + "http://acct-country-de-city-berlin-sid-abc123:secret@gate.nodemaven.com:8080", + ) + assert NodeMavenProvider().get_location() == "de" + + +def test_geo_proxy_url_replaces_country_and_drops_its_locations( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, + "PROXY_URL", + "http://acct-country-us-region-texas-city-austin-sid-old-ttl-10m:secret@gate.nodemaven.com:8080", + ) + + result = NodeMavenProvider().get_geo_proxy_url("gb") + + assert result == "http://acct-ttl-10m-country-gb:secret@gate.nodemaven.com:8080" + + +def test_geo_proxy_url_same_country_keeps_region( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, + "PROXY_URL", + "http://acct-country-us-region-texas:secret@gate.nodemaven.com:8080", + ) + + result = NodeMavenProvider().get_geo_proxy_url("US") + + assert ( + result == "http://acct-country-us-region-texas:secret@gate.nodemaven.com:8080" + ) + + +def test_geo_proxy_url_without_country_keeps_base_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + + assert NodeMavenProvider().get_geo_proxy_url(None) == _URL + assert NodeMavenProvider().get_geo_proxy_url("") == _URL + + +def test_sticky_proxy_url_composes_country_and_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + + result = NodeMavenProvider().get_sticky_proxy_url("location-123", "gb") + + assert result == ( + "http://acct-country-gb-sid-location123:secret@gate.nodemaven.com:8080" + ) + + +def test_sticky_proxy_url_replaces_existing_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, + "PROXY_URL", + "http://acct-country-us-sid-old:secret@gate.nodemaven.com:8080", + ) + + result = NodeMavenProvider().get_sticky_proxy_url("new1") + + assert result == "http://acct-country-us-sid-new1:secret@gate.nodemaven.com:8080" + + +def test_sticky_proxy_url_rejects_empty_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + + with pytest.raises(ValueError): + NodeMavenProvider().get_sticky_proxy_url("---") + + +def test_password_with_reserved_characters_round_trips( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, + "PROXY_URL", + "http://acct-country-us:p%40ss%2Fw@gate.nodemaven.com:8080", + ) + + result = NodeMavenProvider().get_sticky_proxy_url("s1") + + assert result == "http://acct-country-us-sid-s1:p%40ss%2Fw@gate.nodemaven.com:8080" + + +def test_no_country_param_yields_empty_location( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + Config, "PROXY_URL", "http://acct:secret@gate.nodemaven.com:8080" + ) + assert NodeMavenProvider().get_location() == "" + + +def test_unconfigured_returns_none() -> None: + provider = NodeMavenProvider() + + assert provider.get_proxy_url() is None + assert provider.get_requests_proxies() is None + assert provider.get_playwright_proxy() is None + assert provider.get_location() == "" + assert provider.get_sticky_proxy_url("s1") is None + + +def test_playwright_proxy_from_base_parse(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_URL", _URL) + + assert NodeMavenProvider().get_playwright_proxy() == { + "server": "http://gate.nodemaven.com:8080", + "username": "acct-country-us", + "password": "secret", + } diff --git a/surfsense_backend/tests/unit/utils/proxy/test_registry.py b/surfsense_backend/tests/unit/utils/proxy/test_registry.py index 5a6bde403f..940c5a43b7 100644 --- a/surfsense_backend/tests/unit/utils/proxy/test_registry.py +++ b/surfsense_backend/tests/unit/utils/proxy/test_registry.py @@ -1,7 +1,7 @@ """Unit tests for proxy provider selection. -``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` (the default) -and ``dataimpulse`` are registered, and unknown values still warn and fall back +``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` (the default), +``dataimpulse`` and ``nodemaven`` are registered, and unknown values still warn and fall back to the default. """ @@ -11,6 +11,7 @@ from app.utils.proxy import registry from app.utils.proxy.providers.custom import CustomProxyProvider from app.utils.proxy.providers.dataimpulse import DataImpulseProvider +from app.utils.proxy.providers.nodemaven import NodeMavenProvider pytestmark = pytest.mark.unit @@ -31,6 +32,11 @@ def test_resolves_dataimpulse(monkeypatch: pytest.MonkeyPatch) -> None: assert isinstance(registry.get_active_provider(), DataImpulseProvider) +def test_resolves_nodemaven(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Config, "PROXY_PROVIDER", "nodemaven") + assert isinstance(registry.get_active_provider(), NodeMavenProvider) + + def test_unknown_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(Config, "PROXY_PROVIDER", "does_not_exist") assert isinstance(registry.get_active_provider(), CustomProxyProvider)