Added no_proxy os variable bypass so urllib.requests acts as expected

src/meshctrl/files.py

@@ -4,9 +4,27 @@ from . import exceptions
 from . import util
 import asyncio
 import json
-import urllib
+import importlib
+import importlib.util
 import shutil
 
+# import urllib
+# import urllib.request
+import urllib.parse
+old_parse = urllib.parse
+# Default proxy handler uses OS defined no_proxy in order to be helpful. This is unhelpful for our usecase. Monkey patch out proxy getting functions, but don't effect the user's urllib instance.
+spec = importlib.util.find_spec('urllib')
+urllib = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(urllib)
+spec = importlib.util.find_spec('urllib.request')
+urllib.request = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(urllib.request)
+urllib.parse = old_parse
+urllib.request.getproxies_environment = lambda: {}
+urllib.request.getproxies_registry = lambda: {}
+urllib.request.getproxies_macosx_sysconf = lambda: {}
+urllib.request.getproxies = lambda: {}
+
 class Files(tunnel.Tunnel):
     def __init__(self, session, node):
         super().__init__(session, node.nodeid, constants.Protocol.FILES)
@@ -23,11 +41,12 @@ class Files(tunnel.Tunnel):
         if self._session._proxy is not None:
             # We don't know which protocol the user is going to use, but we only need support one at a time, so just assume both
             proxies = {
-                "http_proxy": self._session._proxy,
+                "http": self._session._proxy,
-                "https_proxy": self._session._proxy
+                "https": self._session._proxy,
+                "no": ""
             }
         self._proxy_handler = urllib.request.ProxyHandler(proxies=proxies)
-        self._http_opener = urllib.request.build_opener(self._proxy_handler, urllib.request.HTTPSHandler(context=self._ssl_context))
+        self._http_opener = urllib.request.build_opener(self._proxy_handler, urllib.request.HTTPSHandler(context=self._session._ssl_context))
 
 
     def _get_request_id(self):

src/meshctrl/session.py

@@ -124,15 +124,17 @@ class Session(object):
         self._message_queue = asyncio.Queue()
         self._send_task = None
         self._listen_task = None
+        self._ssl_context = None
+        if self._ignore_ssl:
+            self._ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+            self._ssl_context.check_hostname = False
+            self._ssl_context.verify_mode = ssl.CERT_NONE
 
     async def _main_loop(self):
         try:
             options = {}
-            if self._ignore_ssl:
+            if self._ssl_context is not None:
-                ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+                options["ssl"] = self._ssl_context
-                ssl_context.check_hostname = False
-                ssl_context.verify_mode = ssl.CERT_NONE
-                options = { "ssl": ssl_context }
 
             headers = websockets.datastructures.Headers()
 
@@ -215,11 +217,14 @@ class Session(object):
         return self._command_id
 
     async def close(self):
-        self._main_loop_task.cancel()
         try:
-            await self._main_loop_task
+            await asyncio.gather(*[tunnel.close() for name, tunnel in self._file_tunnels.items()])
-        except asyncio.CancelledError:
+        finally:
-            pass
+            self._main_loop_task.cancel()
+            try:
+                await self._main_loop_task
+            except asyncio.CancelledError:
+                pass
 
     @util._check_socket
     async def __aenter__(self):

src/meshctrl/tunnel.py

@@ -27,11 +27,6 @@ class Tunnel(object):
         self._message_queue = asyncio.Queue()
         self._send_task = None
         self._listen_task = None
-        self._ssl_context = None
-        if self._session._ignore_ssl:
-            self._ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-            self._ssl_context.check_hostname = False
-            self._ssl_context.verify_mode = ssl.CERT_NONE
 
     async def close(self):
         self._main_loop_task.cancel()
@@ -53,8 +48,8 @@ class Tunnel(object):
             self._authcookie = await self._session._send_command_no_response_id({ "action":"authcookie" })
 
             options = {}
-            if self._ssl_context is not None:
+            if self._session._ssl_context is not None:
-                options = { "ssl": self._ssl_context }
+                options["ssl"] = self._session._ssl_context
 
             if (len(self.node_id.split('/')) != 3):
                 self.node_id = f"node/{self._session._currentDomain or ""}/{self.node_id}"
@@ -82,7 +77,6 @@ class Tunnel(object):
                 except* websockets.ConnectionClosed as e:
                     self._socket_open.clear()
                     if not self.auto_reconnect:
-                        self.alive = False
                         raise
         except* Exception as eg:
             self.alive = False

tests/test_files.py

@@ -53,6 +53,18 @@ async def test_commands(env):
         finally:
             assert (await admin_session.remove_device_group(mesh.meshid, timeout=10)), "Failed to remove device group"
 
+async def test_os_proxy_bypass():
+    os.environ["no_proxy"] = "*"
+    import urllib
+    import urllib.request
+    os_proxies = urllib.request.getproxies()
+    meshctrl_proxies = meshctrl.files.urllib.request.getproxies()
+    print(f"os_proxies: {os_proxies}")
+    print(f"meshctrl_proxies: {meshctrl_proxies}")
+    assert meshctrl_proxies.get("no", None) == None, "Meshctrl is using system proxies"
+    assert os_proxies.get("no", None) == "*", "System is using meshctrl proxies"
+    assert os_proxies != meshctrl_proxies, "Override didn't work"
+
 async def test_upload_download(env):
     async with meshctrl.Session("wss://" + env.dockerurl, user="admin", password=env.users["admin"], ignore_ssl=True, proxy=env.proxyurl) as admin_session:
         mesh = await admin_session.add_device_group("test", description="This is a test group", amtonly=False, features=0, consent=0, timeout=10)
@@ -70,7 +82,7 @@ async def test_upload_download(env):
                     else:
                         break
 
-                randdata = random.randbytes(2000000)
+                randdata = random.randbytes(20000000)
                 upfilestream = io.BytesIO(randdata)
                 downfilestream = io.BytesIO()
 
@@ -99,7 +111,7 @@ async def test_upload_download(env):
                     start = time.perf_counter()
                     r = await files.download(f"{pwd}/test", downfilestream, skip_ws_attempt=True, timeout=5)
                     print("\ninfo files_download: {}\n".format(r))
-                    assert r["result"] == True, "Domnload failed"
+                    assert r["result"] == True, "Download failed"
                     assert r["size"] == len(randdata), "Downloaded wrong number of bytes"
                     print(f"http download time: {time.perf_counter()-start}")
 
@@ -110,7 +122,7 @@ async def test_upload_download(env):
                     start = time.perf_counter()
                     r = await files.download(f"{pwd}/test", downfilestream, skip_http_attempt=True, timeout=5)
                     print("\ninfo files_download: {}\n".format(r))
-                    assert r["result"] == True, "Domnload failed"
+                    assert r["result"] == True, "Download failed"
                     assert r["size"] == len(randdata), "Downloaded wrong number of bytes"
                     print(f"ws download time: {time.perf_counter()-start}")
 

tests/test_session.py

@@ -278,7 +278,7 @@ async def test_mesh_device(env):
 
             assert await admin_session.move_to_device_group([agent.nodeid], mesh.name, isname=True, timeout=5), "Failed to move mesh to new device group by name"
 
-            # For now, this expects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
+            # For now, this expe namects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
             # assert "meshagent" in (await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)
@@ -408,7 +408,7 @@ async def test_session_files(env):
                         break
                 pwd = (await admin_session.run_command(agent.nodeid, "pwd", timeout=10))[agent.nodeid]["result"].strip()
 
-                randdata = random.randbytes(2000000)
+                randdata = random.randbytes(20000000)
                 upfilestream = io.BytesIO(randdata)
                 downfilestream = io.BytesIO()
                 os.makedirs(os.path.join(thisdir, "data"), exist_ok=True)