Added test for missing device

Added user agent to ws connection.

CHANGELOG.rst

@@ -2,6 +2,33 @@
 Changelog
 =========
 
+version 1.2.2
+=============
+
+Improvements:
+	* Added user agent to websocket headers
+
+Bugs:
+	* Fixed library's __version__ implementation
+	* Fixed data from certain devices not showing up due to overloading websocket packet sizes
+
+version 1.2.1
+=============
+
+Bugs:
+	* Fixed handling of meshcentral's list_devices return with details=True
+
+version 1.2.0
+=============
+
+Bugs:
+	* Fixed agent sometimes being None causing an oxception
+	* Fixed bad code in device_open_url
+
+Features:
+	* Changed websockets version to 15. This now uses the proxy implemention from that library, instead of the previous hack.
+	* Added lastaddr and lastconnect to list_devices API
+
 version 1.1.2
 =============
 Bugs:

src/meshctrl/__init__.py

@@ -8,7 +8,7 @@ else:
 
 try:
     # Change here if project is renamed and does not equal the package name
-    dist_name = "meshctrl"
+    dist_name = "libmeshctrl"
     __version__ = version(dist_name)
 except PackageNotFoundError:  # pragma: no cover
     __version__ = "unknown"
@@ -24,4 +24,4 @@ from . import files
 from . import exceptions
 from . import device
 from . import mesh
-from . import user_group
+from . import user_group

src/meshctrl/device.py

@@ -59,7 +59,7 @@ class Device(object):
     def __init__(self, nodeid, session, agent=None,
                        name=None, desc=None, description=None,
                        tags=None, users=None,
-                       agct=None, created_at=None, 
+                       agct=None, created_at=None,
                        rname=None, computer_name=None, icon=constants.Icon.desktop,
                        mesh=None, mtype=None, meshtype=None, groupname=None, meshname=None,
                        domain=None, host=None, ip=None, conn=None, connected=None,
@@ -71,7 +71,7 @@ class Device(object):
         if links is None:
             links = {}
         self.links = links
-        if ("ver" in agent):
+        if agent and "ver" in agent:
             agent = {
                 "version": agent["ver"],
                 "id": agent["id"],

src/meshctrl/session.py

@@ -10,6 +10,8 @@ import io
 import ssl
 import urllib
 from python_socks.async_.asyncio import Proxy
+from platform import python_version
+from . import __version__
 from . import constants
 from . import exceptions
 from . import util
@@ -45,7 +47,8 @@ class Session(object):
         closed (asyncio.Event): Event that occurs when the session closes permanently
     '''
 
-    def __init__(self, url, user=None, domain=None, password=None, loginkey=None, proxy=None, token=None, ignore_ssl=False, auto_reconnect=False):
+    def __init__(self, url, user=None, domain=None, password=None, loginkey=None, proxy=None, token=None, ignore_ssl=False, auto_reconnect=False, user_agent_header=None):
+        default_user_agent_header = f"Python/{python_version()} websockets/{websockets.__version__} pylibmeshctrl/{__version__}" 
         parsed = urllib.parse.urlparse(url)
 
         if parsed.scheme not in ("wss", "ws"):
@@ -106,6 +109,10 @@ class Session(object):
         self._file_tunnels = {}
         self._ignore_ssl = ignore_ssl
         self.auto_reconnect = auto_reconnect
+        if user_agent_header:
+            self.user_agent_header = user_agent_header
+        else:
+            self.user_agent_header = default_user_agent_header
 
         self._eventer = util.Eventer()
 
@@ -144,7 +151,7 @@ class Session(object):
 
 
             options["additional_headers"] = headers
-            async for websocket in websockets.asyncio.client.connect(self.url, proxy=self._proxy, process_exception=util._process_websocket_exception, **options):
+            async for websocket in websockets.asyncio.client.connect(self.url, proxy=self._proxy, process_exception=util._process_websocket_exception, max_size=None, user_agent_header=self.user_agent_header, **options):
                 self.alive = True
                 self._socket_open.set()
                 try:
@@ -177,7 +184,7 @@ class Session(object):
     async def _listen_data_task(self, websocket):
         async for message in websocket:
             await self._eventer.emit("raw", message)
-            # Meshcentral does pong wrong and breaks our parsing, so fix it here.
+            # Meshcentral does pong wrong and breaks our parsing, so fix it here. This is fixed now, but we want compatibility with old versions.
             if message == '{action:"pong"}':
                 message = '{"action":"pong"}'
 
@@ -478,10 +485,21 @@ class Session(object):
         if "result" in res0:
             raise exceptions.ServerError(res0["result"])
         if details:
-            nodes = json.loads(res0["data"])
+            nodes = res0["data"]
+            # Accept any number of nested strings, meshcentral is odd
+            while True:
+                try:
+                    nodes = json.loads(nodes)
+                except TypeError:
+                    break
+
             for node in nodes:
                 if node["node"].get("meshid", None):
                     node["node"]["mesh"] = mesh.Mesh(node["node"].get("meshid"), self)
+                if "lastConnect" in node and isinstance(node["lastConnect"], dict):
+                    node["node"]["lastconnect"] = node["lastConnect"].get("time")
+                    node["node"]["lastaddr"] = node["lastConnect"].get("addr")
+                    del node["lastConnect"]
                 details = {}
                 for key, val in node.items():
                     if key != "node":
@@ -1455,7 +1473,7 @@ class Session(object):
                     return nid
 
         result = {n: {"complete": False, "result": [], "command": command} for n in nodeids}
-        async def _():
+        async def _console():
             async for event in self.events({"action": "msg", "type": "console"}):
                 node = match_nodeid(event["nodeid"], nodeids)
                 if node:
@@ -1467,34 +1485,58 @@ class Session(object):
                     elif (event["value"].startswith("Run commands")):
                         continue
                     result[node]["result"].append(event["value"])
-        async def __(command):
+
+        # We create this task AFTER getting the first message, but I don't feel like implementing this twice, so we'll pass in the first message and have it parsed immediately
+        async def _reply(responseid, start_data=None):
+            # Returns True when all results are in, Falsey otherwise
+            def _parse_event(event):
+                node = match_nodeid(event["nodeid"], nodeids)
+                if node:
+                    result.setdefault(node, {})["complete"] = True
+                    result[node]["result"].append(event["result"])
+                    if all(_["complete"] for key, _ in result.items()):
+                        return True
+
+            if start_data is not None:
+                if _parse_event(start_data):
+                    return
+            async for event in self.events({"action": "msg", "type": "runcommands", "responseid": responseid}):
+                if _parse_event(event):
+                    break
+
+        async def __(command, tg, tasks):
             data = await self._send_command(command, "run_command", timeout=timeout)
 
-            if data.get("result", "ok").lower() != "ok":
+            if data.get("type", None) != "runcommands" and data.get("result", "ok").lower() != "ok":
                 raise exceptions.ServerError(data["result"])
-        
-        expect_response = False
-        if not ignore_output:
-            userid = (await self.user_info())["_id"]
-            for n in nodeids:
-                device_info = await self.device_info(n, timeout=timeout)
-                try:
-                    permissions = device_info.mesh.links.get(userid, {}).get("rights",constants.DeviceRights.norights)\
-                    # This should work for device rights, but it only seems to work for mesh rights. Not sure why, but I can't get the events to show up when the user only has individual device rights
-                    # |device_info.get("links", {}).get(userid, {}).get("rights", constants.DeviceRights.norights)
-                    # If we don't have agentconsole rights, we won't be able te read the output, so fill in blanks on this node
-                    if not permissions&constants.DeviceRights.agentconsole:
-                        result[n]["complete"] = True
-                    else:
-                        expect_response = True
-                except AttributeError:
-                    result[n]["complete"] = True
+            elif data.get("type", None) != "runcommands" and data.get("result", "ok").lower() == "ok":
+                expect_response = False
+                console_task = tg.create_task(asyncio.wait_for(_console(), timeout=timeout))
+                if not ignore_output:
+                    userid = (await self.user_info())["_id"]
+                    for n in nodeids:
+                        device_info = await self.device_info(n, timeout=timeout)
+                        try:
+                            permissions = device_info.mesh.links.get(userid, {}).get("rights",constants.DeviceRights.norights)\
+                            # This should work for device rights, but it only seems to work for mesh rights. Not sure why, but I can't get the events to show up when the user only has individual device rights
+                            # |device_info.get("links", {}).get(userid, {}).get("rights", constants.DeviceRights.norights)
+                            # If we don't have agentconsole rights, we won't be able te read the output, so fill in blanks on this node
+                            if not permissions&constants.DeviceRights.agentconsole:
+                                result[n]["complete"] = True
+                            else:
+                                expect_response = True
+                        except AttributeError:
+                            result[n]["complete"] = True
+                if expect_response:
+                    tasks.append(console_task)
+                else:
+                    console_task.cancel()
+            elif data.get("type", None) == "runcommands" and not ignore_output:
+                tasks.append(tg.create_task(asyncio.wait_for(_reply(data["responseid"], start_data=data), timeout=timeout)))
 
         tasks = []
         async with asyncio.TaskGroup() as tg:
-            if expect_response:
-                tasks.append(tg.create_task(asyncio.wait_for(_(), timeout=timeout)))
-            tasks.append(tg.create_task(__({ "action": 'runcommands', "nodeids": nodeids, "type": (2 if powershell else 0), "cmds": command, "runAsUser": runAsUser })))
+            tasks.append(tg.create_task(__({ "action": 'runcommands', "nodeids": nodeids, "type": (2 if powershell else 0), "cmds": command, "runAsUser": runAsUser, "reply": not ignore_output}, tg, tasks)))
 
         return {n: v | {"result": "".join(v["result"])} for n,v in result.items()}
 
@@ -1737,10 +1779,11 @@ class Session(object):
             tasks.append(tg.create_task(asyncio.wait_for(_(), timeout=timeout)))
             tasks.append({ "action": 'msg', "type": 'openUrl', "nodeid": nodeid, "url": url }, "device_open_url", timeout=timeout)
 
+        
+        success = tasks[0].result()
         res = tasks[1].result()
-        success = tasks[2].result()
 
-        if data.get("result", "ok").lower() != "ok":
+        if res.get("result", "ok").lower() != "ok":
             raise exceptions.ServerError(data["result"])
 
         if not success:
@@ -1964,4 +2007,4 @@ class _FileExplorerWrapper:
         return await self._files.__aenter__()
 
     async def __aexit__(self, exc_t, exc_v, exc_tb):
-        return await self._files.__aexit__(exc_t, exc_v, exc_tb)
+        return await self._files.__aexit__(exc_t, exc_v, exc_tb)

tests/environment/client.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12
+FROM python:3.13
 WORKDIR /usr/local/app
 
 # Install the application dependencies

tests/environment/meshcentral.dockerfile

@@ -1,8 +1,8 @@
-FROM ghcr.io/ylianst/meshcentral:latest
+FROM ghcr.io/ylianst/meshcentral:1.1.50
 RUN apk add curl
 RUN apk add python3
 WORKDIR /opt/meshcentral/
 COPY ./scripts/meshcentral ./scripts
 COPY ./config/meshcentral/data /opt/meshcentral/meshcentral-data
 COPY ./config/meshcentral/overrides /opt/meshcentral/meshcentral
-CMD ["python3", "/opt/meshcentral/scripts/create_users.py"]
+ENTRYPOINT ["python3", "/opt/meshcentral/scripts/create_users.py"]

tests/environment/scripts/meshcentral/create_users.py

@@ -7,9 +7,9 @@ thisdir = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(thisdir, "users.json")) as infile:
     users = json.load(infile)
 for username, password in users.items():
-    subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--createaccount", username, "--pass", password, "--name", username])
+    print(subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--createaccount", username, "--pass", password, "--name", username]))
 
 
-subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--adminaccount", "admin"])
+print(subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--adminaccount", "admin"]))
 
-subprocess.call(["bash", "/opt/meshcentral/startup.sh"])
+subprocess.call(["bash", "/opt/meshcentral/entrypoint.sh"])

tests/test_files.py

@@ -120,7 +120,7 @@ async def test_upload_download(env):
                     downfilestream.seek(0)
 
                     start = time.perf_counter()
-                    r = await files.download(f"{pwd}/test", downfilestream, skip_http_attempt=True, timeout=5)
+                    r = await files.download(f"{pwd}/test", downfilestream, skip_http_attempt=True, timeout=20)
                     print("\ninfo files_download: {}\n".format(r))
                     assert r["result"] == True, "Download failed"
                     assert r["size"] == len(randdata), "Downloaded wrong number of bytes"

tests/test_sanity.py

@@ -9,16 +9,7 @@ import requests
 
 async def test_sanity(env):
     async with meshctrl.Session(env.mcurl, user="unprivileged", password=env.users["unprivileged"], ignore_ssl=True) as s:
-        got_pong = asyncio.Event()
-        async def _():
-            async for raw in s.raw_messages():
-                if raw == '{action:"pong"}':
-                    got_pong.set()
-                    break
-        ping_task = None
         async with asyncio.TaskGroup() as tg:
-            tg.create_task(asyncio.wait_for(_(), timeout=5))
-            tg.create_task(asyncio.wait_for(got_pong.wait(), timeout=5))
             ping_task = tg.create_task(s.ping(timeout=10))
         print("\ninfo ping: {}\n".format(ping_task.result()))
         print("\ninfo user_info: {}\n".format(await s.user_info()))

tests/test_session.py

@@ -251,7 +251,15 @@ async def test_mesh_device(env):
             assert "Run commands completed." not in r[agent2.nodeid]["result"], "Didn't parse run command ending correctly"
             assert "meshagent" in (await privileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
 
-            # Test run commands with ndividual device permissions
+            # Test run_commands missing device
+            try:
+                await admin_session.run_command([agent.nodeid, "notanid"], "ls", timeout=10)
+            except* (meshctrl.exceptions.ServerError, ValueError):
+                pass
+            else:
+                raise Exception("Run command on a device that doesn't exist did not raise an exception")
+
+            # Test run commands with individual device permissions
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)
             except* (meshctrl.exceptions.ServerError, ValueError):
@@ -266,7 +274,7 @@ async def test_mesh_device(env):
             else:
                 raise Exception("Unprivileged user has access to device it should not")
 
-            assert (await admin_session.add_users_to_device((await unprivileged_session.user_info())["_id"], agent.nodeid, meshctrl.constants.MeshRights.norights)), "Failed to add user to device"
+            assert (await admin_session.add_users_to_device((await unprivileged_session.user_info())["_id"], agent.nodeid, meshctrl.constants.DeviceRights.norights)), "Failed to add user to device"
 
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", ignore_output=True, timeout=10)
@@ -284,12 +292,14 @@ async def test_mesh_device(env):
 
             assert r.links[(await unprivileged_session.user_info())["_id"]]["rights"] == meshctrl.constants.DeviceRights.norights, "Unprivileged user has too many rights!"
 
-            assert (await admin_session.add_users_to_device([(await unprivileged_session.user_info())["_id"]], agent.nodeid, meshctrl.constants.DeviceRights.remotecontrol|meshctrl.constants.DeviceRights.agentconsole|meshctrl.constants.DeviceRights.remotecommands)), "Failed to modify user's permissions"
+            assert (await admin_session.add_users_to_device([(await unprivileged_session.user_info())["_id"]], agent.nodeid, meshctrl.constants.DeviceRights.fullrights)), "Failed to modify user's permissions"
 
-            assert (await unprivileged_session.device_info(agent.nodeid, timeout=10)).links[(await unprivileged_session.user_info())["_id"]]["rights"] == meshctrl.constants.DeviceRights.remotecontrol|meshctrl.constants.DeviceRights.agentconsole|meshctrl.constants.DeviceRights.remotecommands, "Adding permissions did not update unprivileged user."
+            assert (await unprivileged_session.device_info(agent.nodeid, timeout=10)).links[(await unprivileged_session.user_info())["_id"]]["rights"] == meshctrl.constants.DeviceRights.fullrights, "Adding permissions did not update unprivileged user."
 
-            # For now, this expects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
+            # For now, this expects no response. If we ever figure out why the server isn't sending console information to us when it should, fix this.
             # assert "meshagent" in (await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
+            # Meshcentral has a 10 second cache on user perms.
+            #await asyncio.sleep(15)
             await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)
 
             assert await admin_session.move_to_device_group(agent.nodeid, mesh2.meshid, timeout=5), "Failed to move mesh to new device group"
@@ -303,7 +313,7 @@ async def test_mesh_device(env):
 
             assert await admin_session.move_to_device_group([agent.nodeid], mesh.name, isname=True, timeout=5), "Failed to move mesh to new device group by name"
 
-            # For now, this expe namects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
+            # For now, this expects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
             # assert "meshagent" in (await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)