feat(proxy): add NodeMaven provider

surfsense_backend/.env.example

@@ -436,7 +436,7 @@ TURNSTILE_SECRET_KEY=
 
 
 # Proxy provider selection. Selects a ProxyProvider implementation registered in
-# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse".
+# app/utils/proxy/registry.py. Default: "custom". Options: "custom", "dataimpulse", "nodemaven".
 # PROXY_PROVIDER=custom
 
 # Proxy endpoint(s), shared across providers — PROXY_PROVIDER picks the behavior,
@@ -447,6 +447,8 @@ TURNSTILE_SECRET_KEY=
 #   provider). For "dataimpulse", country is a "__cr.<country>" username suffix the
 #   provider parses for geoip-match, e.g.:
 #   PROXY_URL=http://your_token__cr.us:your_password@gw.dataimpulse.com:823
+#   For "nodemaven", country is a "-country-<country>" username parameter, e.g.:
+#   PROXY_URL=http://your_login-country-us:your_password@gate.nodemaven.com:8080
 # PROXY_URLS: a comma-separated pool the "custom" provider rotates client-side
 #   (cyclic); server-side-rotating gateways ignore it.
 # PROXY_URL=http://user:pass@host:port

surfsense_backend/app/utils/proxy/providers/nodemaven.py

@@ -0,0 +1,99 @@
+"""NodeMaven residential / rotating proxy provider.
+
+Takes the shared ``PROXY_URL`` env, like ``custom`` and ``dataimpulse``. What it
+adds is NodeMaven's username syntax, ``<login>-<param>-<value>-...``:
+
+* :meth:`get_location` reads the ``-country-<cc>`` parameter so the crawler's
+  geoip-match can align the browser locale with the exit country.
+* Country routing and sticky sessions rewrite the username with
+  ``-country-<cc>`` and ``-sid-<id>``.
+
+Rotation happens server-side (a fresh exit IP per connection without ``sid``),
+so this is NOT :pyattr:`~ProxyProvider.is_pool_backed`.
+
+Example URL::
+
+    http://your_login-country-us:your_password@gate.nodemaven.com:8080
+
+``region``, ``city`` and ``isp`` belong to a country, so they are dropped when
+the country is changed and kept otherwise; ``ttl`` and other parameters are
+kept as written.
+"""
+
+import re
+from urllib.parse import quote, unquote, urlsplit, urlunsplit
+
+from app.config import Config
+from app.utils.proxy.base import ProxyProvider
+
+_COUNTRY_RE = re.compile(r"-country-([A-Za-z_]+)")
+_LOCATION_RE = re.compile(r"-(?:country|region|city|isp)-[A-Za-z0-9_]+")
+_SESSION_RE = re.compile(r"-sid-[A-Za-z0-9]+")
+
+
+def _safe_session_id(session_id: str) -> str:
+    # "-" separates parameters in the username, so a session id is alphanumeric.
+    safe_id = re.sub(r"[^A-Za-z0-9]", "", session_id)
+    if not safe_id:
+        raise ValueError("session_id must contain at least one letter or digit")
+    return safe_id
+
+
+def _safe_country(country: str | None) -> str | None:
+    if country is None:
+        return None
+    safe_country = re.sub(r"[^a-z]", "", country.lower())
+    return safe_country or None
+
+
+class NodeMavenProvider(ProxyProvider):
+    """Provider for a NodeMaven proxy URL in the shared ``PROXY_URL`` env."""
+
+    name = "nodemaven"
+
+    def get_proxy_url(self) -> str | None:
+        url = (Config.PROXY_URL or "").strip()
+        return url or None
+
+    def get_location(self) -> str:
+        """Country from the ``-country-<cc>`` username parameter, or ``""``."""
+        url = self.get_proxy_url()
+        if not url:
+            return ""
+        match = _COUNTRY_RE.search(unquote(urlsplit(url).username or ""))
+        return match.group(1).lower() if match else ""
+
+    def _rewrite_proxy_url(
+        self, *, country: str | None = None, session_id: str | None = None
+    ) -> str | None:
+        url = self.get_proxy_url()
+        if not url:
+            return None
+        parts = urlsplit(url)
+        username = _SESSION_RE.sub("", unquote(parts.username or ""))
+        safe_country = _safe_country(country)
+        if safe_country is not None and safe_country != self.get_location():
+            username = _LOCATION_RE.sub("", username) + f"-country-{safe_country}"
+        if session_id is not None:
+            username += f"-sid-{_safe_session_id(session_id)}"
+        userinfo = quote(username, safe="")
+        if parts.password is not None:
+            userinfo += f":{quote(unquote(parts.password), safe='')}"
+        netloc = f"{userinfo}@{parts.hostname or ''}"
+        if parts.port:
+            netloc += f":{parts.port}"
+        return urlunsplit(
+            (parts.scheme, netloc, parts.path, parts.query, parts.fragment)
+        )
+
+    def get_geo_proxy_url(self, country: str | None = None) -> str | None:
+        """Return the configured URL routed to ``country`` when one is given."""
+        if not _safe_country(country):
+            return self.get_proxy_url()
+        return self._rewrite_proxy_url(country=country)
+
+    def get_sticky_proxy_url(
+        self, session_id: str, country: str | None = None
+    ) -> str | None:
+        """Return the configured URL with a session id and optional country."""
+        return self._rewrite_proxy_url(country=country, session_id=session_id)

surfsense_backend/app/utils/proxy/registry.py

@@ -11,13 +11,15 @@
 from app.utils.proxy.base import ProxyProvider
 from app.utils.proxy.providers.custom import CustomProxyProvider
 from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
+from app.utils.proxy.providers.nodemaven import NodeMavenProvider
 
 logger = logging.getLogger(__name__)
 
 # Registered proxy providers, keyed by their ``name``.
 _PROVIDERS: dict[str, type[ProxyProvider]] = {
     CustomProxyProvider.name: CustomProxyProvider,
     DataImpulseProvider.name: DataImpulseProvider,
+    NodeMavenProvider.name: NodeMavenProvider,
 }
 
 # BYO ``custom`` is the neutral default: it needs no vendor and returns no proxy

surfsense_backend/tests/unit/utils/proxy/test_nodemaven_provider.py

@@ -0,0 +1,160 @@
+"""Unit tests for the NodeMavenProvider.
+
+Takes a single full URL (like ``custom``); the vendor-specific bit is the
+``-country-<cc>`` / ``-sid-<id>`` username syntax used for geoip-match, country
+routing and sticky sessions.
+"""
+
+import pytest
+
+from app.config import Config
+from app.utils.proxy.providers.nodemaven import NodeMavenProvider
+
+pytestmark = pytest.mark.unit
+
+_URL = "http://acct-country-us:secret@gate.nodemaven.com:8080"
+
+
+@pytest.fixture(autouse=True)
+def _clear_env(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", None)
+
+
+def test_returns_configured_url(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+    provider = NodeMavenProvider()
+
+    assert provider.is_pool_backed is False
+    assert provider.get_proxy_url() == _URL
+    assert provider.get_requests_proxies() == {"http": _URL, "https": _URL}
+
+
+def test_location_parsed_from_country_param(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+    assert NodeMavenProvider().get_location() == "us"
+
+
+def test_location_stops_at_next_param(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://acct-country-de-city-berlin-sid-abc123:secret@gate.nodemaven.com:8080",
+    )
+    assert NodeMavenProvider().get_location() == "de"
+
+
+def test_geo_proxy_url_replaces_country_and_drops_its_locations(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://acct-country-us-region-texas-city-austin-sid-old-ttl-10m:secret@gate.nodemaven.com:8080",
+    )
+
+    result = NodeMavenProvider().get_geo_proxy_url("gb")
+
+    assert result == "http://acct-ttl-10m-country-gb:secret@gate.nodemaven.com:8080"
+
+
+def test_geo_proxy_url_same_country_keeps_region(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://acct-country-us-region-texas:secret@gate.nodemaven.com:8080",
+    )
+
+    result = NodeMavenProvider().get_geo_proxy_url("US")
+
+    assert (
+        result == "http://acct-country-us-region-texas:secret@gate.nodemaven.com:8080"
+    )
+
+
+def test_geo_proxy_url_without_country_keeps_base_url(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    assert NodeMavenProvider().get_geo_proxy_url(None) == _URL
+    assert NodeMavenProvider().get_geo_proxy_url("") == _URL
+
+
+def test_sticky_proxy_url_composes_country_and_session(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    result = NodeMavenProvider().get_sticky_proxy_url("location-123", "gb")
+
+    assert result == (
+        "http://acct-country-gb-sid-location123:secret@gate.nodemaven.com:8080"
+    )
+
+
+def test_sticky_proxy_url_replaces_existing_session(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://acct-country-us-sid-old:secret@gate.nodemaven.com:8080",
+    )
+
+    result = NodeMavenProvider().get_sticky_proxy_url("new1")
+
+    assert result == "http://acct-country-us-sid-new1:secret@gate.nodemaven.com:8080"
+
+
+def test_sticky_proxy_url_rejects_empty_session(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    with pytest.raises(ValueError):
+        NodeMavenProvider().get_sticky_proxy_url("---")
+
+
+def test_password_with_reserved_characters_round_trips(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config,
+        "PROXY_URL",
+        "http://acct-country-us:p%40ss%2Fw@gate.nodemaven.com:8080",
+    )
+
+    result = NodeMavenProvider().get_sticky_proxy_url("s1")
+
+    assert result == "http://acct-country-us-sid-s1:p%40ss%2Fw@gate.nodemaven.com:8080"
+
+
+def test_no_country_param_yields_empty_location(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        Config, "PROXY_URL", "http://acct:secret@gate.nodemaven.com:8080"
+    )
+    assert NodeMavenProvider().get_location() == ""
+
+
+def test_unconfigured_returns_none() -> None:
+    provider = NodeMavenProvider()
+
+    assert provider.get_proxy_url() is None
+    assert provider.get_requests_proxies() is None
+    assert provider.get_playwright_proxy() is None
+    assert provider.get_location() == ""
+    assert provider.get_sticky_proxy_url("s1") is None
+
+
+def test_playwright_proxy_from_base_parse(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_URL", _URL)
+
+    assert NodeMavenProvider().get_playwright_proxy() == {
+        "server": "http://gate.nodemaven.com:8080",
+        "username": "acct-country-us",
+        "password": "secret",
+    }

surfsense_backend/tests/unit/utils/proxy/test_registry.py

@@ -1,7 +1,7 @@
 """Unit tests for proxy provider selection.
 
-``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` (the default)
-and ``dataimpulse`` are registered, and unknown values still warn and fall back
+``PROXY_PROVIDER`` selects the single app-wide provider; ``custom`` (the default),
+``dataimpulse`` and ``nodemaven`` are registered, and unknown values still warn and fall back
 to the default.
 """
 
@@ -11,6 +11,7 @@
 from app.utils.proxy import registry
 from app.utils.proxy.providers.custom import CustomProxyProvider
 from app.utils.proxy.providers.dataimpulse import DataImpulseProvider
+from app.utils.proxy.providers.nodemaven import NodeMavenProvider
 
 pytestmark = pytest.mark.unit
 
@@ -31,6 +32,11 @@ def test_resolves_dataimpulse(monkeypatch: pytest.MonkeyPatch) -> None:
     assert isinstance(registry.get_active_provider(), DataImpulseProvider)
 
 
+def test_resolves_nodemaven(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(Config, "PROXY_PROVIDER", "nodemaven")
+    assert isinstance(registry.get_active_provider(), NodeMavenProvider)
+
+
 def test_unknown_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(Config, "PROXY_PROVIDER", "does_not_exist")
     assert isinstance(registry.get_active_provider(), CustomProxyProvider)