Bump version

fix #46

.readthedocs.yml

@@ -19,7 +19,7 @@ formats:
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.13"
 
 python:
   install:

AUTHORS.rst

@@ -3,3 +3,4 @@ Contributors
 ============
 
 * Josiah Baldwin <jbaldwin8889@gmail.com>
+* Daan Selen <https://github.com/DaanSelen>

CHANGELOG.rst

@@ -2,7 +2,93 @@
 Changelog
 =========
 
-Version 0.1
-===========
+version 1.3.3
+=============
 
-Create
+Improvements:
+	* Dependency bumps
+
+Bugs:
+	* Fix run_commands having an issue with ignore_output
+
+version 1.3.2
+=============
+
+Improvements:
+	* Fix race condition that could occur when running `run_command` or `run_console_command`
+
+version 1.3.1
+=============
+
+Improvments:
+	* Basically just everything in 1.3.0, this is a release fix
+
+version 1.3.0
+=============
+
+Improvements:
+	* Improved how run_commands was handled (#51)
+	* Added remove device functionality (#52)
+	* Added run_console_commands functionality (#55)
+
+Bugs:
+	* Silly documentation being wrong (#53)
+
+version 1.2.2
+=============
+
+Improvements:
+	* Added user agent to websocket headers
+
+Bugs:
+	* Fixed library's __version__ implementation
+	* Fixed data from certain devices not showing up due to overloading websocket packet sizes
+
+version 1.2.1
+=============
+
+Bugs:
+	* Fixed handling of meshcentral's list_devices return with details=True
+
+version 1.2.0
+=============
+
+Bugs:
+	* Fixed agent sometimes being None causing an oxception
+	* Fixed bad code in device_open_url
+
+Features:
+	* Changed websockets version to 15. This now uses the proxy implemention from that library, instead of the previous hack.
+	* Added lastaddr and lastconnect to list_devices API
+
+version 1.1.2
+=============
+Bugs:
+	* Fixed semver for requirements. New version of websockets broke this library.
+
+Security:
+	* Updated cryptogaphy to ~44.0.1 to fix ssl vulnerability.
+
+Version 1.1.1
+=============
+Bugs:
+	* Fixed bug when running device_info when user has access to multiple meshes
+
+Version 1.1.0
+=============
+Features:
+	* Added overrides for meshcentral files for testing purposes
+	* Added `users` field to `device` object
+
+Bugs:
+	* Fixed connection errors not raising immediately
+	* Fixed run_commands parsing return from multiple devices incorrectly
+	* Fixed listening to raw not removing its listener correctly
+	* Fixed javascript timecodes not being handled in gnu environments
+	* Changed some fstring formatting that locked the library into python >3.13
+
+
+Version 1.0.0
+=============
+
+First release

README.rst

@@ -38,7 +38,7 @@ Library for remotely interacting with a
 Installation
 ------------
 
-pip install meshctrl
+pip install libmeshctrl
 
 Usage
 -----

docs/requirements.txt

@@ -5,8 +5,8 @@ sphinx>=3.2.1
 sphinx-jinja2-compat>=0.1.1
 sphinx-toolbox>=2.16.0
 # sphinx_rtd_theme
-cffi==1.17.1
-cryptography==43.0.3
-pycparser==2.22
-websockets==13.1
+cffi~=1.17.1
+pycparser~=2.22
 enum_tools
+cryptography~=46.0.5
+websockets~=16.0.0

setup.cfg

@@ -24,7 +24,7 @@ platforms = any
 # Add here all kinds of additional classifiers as defined under
 # https://pypi.org/classifiers/
 classifiers =
-    Development Status :: 4 - Beta
+    Development Status :: 5 - Production/Stable
     Programming Language :: Python
 
 
@@ -44,9 +44,9 @@ python_requires = >=3.8
 # For more information, check out https://semver.org/.
 install_requires =
     importlib-metadata
-    cryptography>=43.0.3
-    websockets>=13.1
-    python-socks[asyncio]
+    cryptography~=46.0.5
+    websockets~=16.0.0
+    python-socks[asyncio]~=2.8.1
 
 
 [options.packages.find]

src/meshctrl/__init__.py

@@ -8,7 +8,7 @@ else:
 
 try:
     # Change here if project is renamed and does not equal the package name
-    dist_name = "meshctrl"
+    dist_name = "libmeshctrl"
     __version__ = version(dist_name)
 except PackageNotFoundError:  # pragma: no cover
     __version__ = "unknown"

src/meshctrl/device.py

@@ -12,6 +12,7 @@ class Device(object):
         name (str|None): Device name as it is shown on the meshcentral server
         description (str|None): Device description as it is shown on the meshcentral server. Also accepted as desc.
         tags (list[str]|None): tags associated with device.
+        users (list[str]|None): latest known usernames which have logged in.
         created_at (datetime.Datetime|int|None): Time at which device mas created. Also accepted as agct.
         computer_name (str|None): Device name as reported from the agent. This may be different from name. Also accepted as rname.
         icon (~meshctrl.constants.Icon): Icon displayed on the website
@@ -38,6 +39,7 @@ class Device(object):
         name (str|None): Device name as it is shown on the meshcentral server
         description (str|None): Device description as it is shown on the meshcentral server.
         tags (list[str]): tags associated with device.
+        users (list[str]): latest known usernames which have logged in.
         computer_name (str|None): Device name as reported from the agent. This may be different from name. Also accepted as rname.
         icon (~meshctrl.constants.Icon): Icon displayed on the website
         mesh (~meshctrl.mesh.Mesh|None): Mesh object under which this device exists. Is None for individual device access.
@@ -56,7 +58,7 @@ class Device(object):
     '''
     def __init__(self, nodeid, session, agent=None,
                        name=None, desc=None, description=None,
-                       tags=None,
+                       tags=None, users=None,
                        agct=None, created_at=None,
                        rname=None, computer_name=None, icon=constants.Icon.desktop,
                        mesh=None, mtype=None, meshtype=None, groupname=None, meshname=None,
@@ -69,7 +71,7 @@ class Device(object):
         if links is None:
             links = {}
         self.links = links
-        if ("ver" in agent):
+        if agent and "ver" in agent:
             agent = {
                 "version": agent["ver"],
                 "id": agent["id"],
@@ -90,13 +92,14 @@ class Device(object):
         self.description = description if description is not None else desc
         self.os_description = os_description if os_description is not None else osdesc
         self.tags = tags if tags is not None else []
+        self.users = users if users is not None else []
         self.details = details if details is not None else {}
 
         created_at = created_at if created_at is not None else agct
         if not isinstance(created_at, datetime.datetime) and created_at is not None:
             try:
                 created_at = datetime.datetime.fromtimestamp(created_at)
-            except OSError:
+            except (OSError, ValueError):
                 # Meshcentral returns in miliseconds, while fromtimestamp, and most of python, expects the argument in seconds. Try seconds frist, then translate from ms if it fails.
                 # This doesn't work for really early timestamps, but I don't expect that to be a problem here.
                 created_at = datetime.datetime.fromtimestamp(created_at/1000.0)
@@ -106,7 +109,7 @@ class Device(object):
         if not isinstance(lastconnect, datetime.datetime) and lastconnect is not None:
             try:
                 lastconnect = datetime.datetime.fromtimestamp(lastconnect)
-            except OSError:
+            except (OSError, ValueError):
                 # Meshcentral returns in miliseconds, while fromtimestamp, and most of python, expects the argument in seconds. Try seconds frist, then translate from ms if it fails.
                 # This doesn't work for really early timestamps, but I don't expect that to be a problem here.
                 lastconnect = datetime.datetime.fromtimestamp(lastconnect/1000.0)
@@ -292,6 +295,23 @@ class Device(object):
          '''
         return await self._session.reset_devices(self.nodeid, timeout=timeout)
 
+    async def remove(self, timeout=None):
+        '''
+        Remove device from MeshCentral
+
+        Args:
+            nodeids (str|list[str]): nodeid(s) of the device(s) that have to be removed
+            timeout (int): duration in seconds to wait for a response before throwing an error
+        
+        Returns:
+            bool: True on success, raise otherwise
+
+        Raises:
+            :py:class:`~meshctrl.exceptions.SocketError`: Info about socket closure
+            asyncio.TimeoutError: Command timed out
+         '''
+        return self._session.remove_devices(self.nodeid, timeout)
+
     async def sleep(self, timeout=None):
         '''
         Sleep device
@@ -340,10 +360,10 @@ class Device(object):
     def __str__(self):
         return f"<Device: nodeid={self.nodeid} name={self.name} description={self.description} computer_name={self.computer_name} icon={self.icon} "\
                f"mesh={self.mesh} meshtype={self.meshtype} meshname={self.meshname} domain={self.domain} host={self.host} ip={self.ip} "\
-               f"tags={self.tags} details={self.details} created_at={self.created_at} lastaddr={self.lastaddr} lastconnect={self.lastconnect} "\
+               f"tags={self.tags} users={self.users} details={self.details} created_at={self.created_at} lastaddr={self.lastaddr} lastconnect={self.lastconnect} "\
                f"connected={self.connected} powered_on={self.powered_on} os_description={self.os_description} links={self.links} _extra_props={self._extra_props}>"
     def __repr__(self):
         return f"Device(nodeid={repr(self.nodeid)}, session={repr(self._session)}, name={repr(self.name)}, description={repr(self.description)}, computer_name={repr(self.computer_name)}, icon={repr(self.icon)}, "\
                f"mesh={repr(self.mesh)}, meshtype={repr(self.meshtype)}, meshname={repr(self.meshname)}, domain={repr(self.domain)}, host={repr(self.host)}, ip={repr(self.ip)}, "\
-               f"tags={repr(self.tags)}, details={repr(self.details)} created_at={repr(self.created_at)} lastaddr={repr(self.lastaddr)} lastconnect={repr(self.lastconnect)} "\
+               f"tags={repr(self.tags)}, users={repr(self.users)}, details={repr(self.details)} created_at={repr(self.created_at)} lastaddr={repr(self.lastaddr)} lastconnect={repr(self.lastconnect)} "\
                f"connected={repr(self.connected)}, powered_on={repr(self.powered_on)}, os_description={repr(self.os_description)}, links={repr(self.links)}, **{repr(self._extra_props)})"

src/meshctrl/exceptions.py

@@ -2,7 +2,9 @@ class MeshCtrlError(Exception):
     """
     Base class for Meshctrl errors
     """
-    pass
+    def __init__(self, message, *args, **kwargs):
+        self.message = message
+        super().__init__(message, *args, **kwargs)
 
 class ServerError(MeshCtrlError):
     """
@@ -25,6 +27,7 @@ class FileTransferError(MeshCtrlError):
     """
     def __init__(self, message, stats):
         self.stats = stats
+        super().__init__(message)
 
 class FileTransferCancelled(FileTransferError):
     """

src/meshctrl/files.py

@@ -157,7 +157,7 @@ class Files(tunnel.Tunnel):
 
     async def rm(self, path, files, recursive=False, timeout=None):
         """
-        Create a directory on the device. This API doesn't error if the file doesn't exist.
+        Remove a set of files or directories from the device. This API doesn't error if the file doesn't exist.
 
         Args:
             path (str): Directory from which to delete files

src/meshctrl/mesh.py

@@ -46,7 +46,7 @@ class Mesh(object):
         if not isinstance(created_at, datetime.datetime) and created_at is not None:
             try:
                 created_at = datetime.datetime.fromtimestamp(created_at)
-            except OSError:
+            except (OSError, ValueError):
                 # Meshcentral returns in miliseconds, while fromtimestamp, and most of python, expects the argument in seconds. Try seconds frist, then translate from ms if it fails.
                 # This doesn't work for really early timestamps, but I don't expect that to be a problem here.
                 created_at = datetime.datetime.fromtimestamp(created_at/1000.0)

src/meshctrl/session.py

@@ -10,6 +10,8 @@ import io
 import ssl
 import urllib
 from python_socks.async_.asyncio import Proxy
+from platform import python_version
+from . import __version__
 from . import constants
 from . import exceptions
 from . import util
@@ -45,7 +47,8 @@ class Session(object):
         closed (asyncio.Event): Event that occurs when the session closes permanently
     '''
 
-    def __init__(self, url, user=None, domain=None, password=None, loginkey=None, proxy=None, token=None, ignore_ssl=False, auto_reconnect=False):
+    def __init__(self, url, user=None, domain=None, password=None, loginkey=None, proxy=None, token=None, ignore_ssl=False, auto_reconnect=False, user_agent_header=None):
+        default_user_agent_header = f"Python/{python_version()} websockets/{websockets.__version__} pylibmeshctrl/{__version__}" 
         parsed = urllib.parse.urlparse(url)
 
         if parsed.scheme not in ("wss", "ws"):
@@ -106,6 +109,10 @@ class Session(object):
         self._file_tunnels = {}
         self._ignore_ssl = ignore_ssl
         self.auto_reconnect = auto_reconnect
+        if user_agent_header:
+            self.user_agent_header = user_agent_header
+        else:
+            self.user_agent_header = default_user_agent_header
 
         self._eventer = util.Eventer()
 
@@ -144,7 +151,7 @@ class Session(object):
 
 
             options["additional_headers"] = headers
-            async for websocket in util.proxy_connect(self.url, proxy_url=self._proxy, process_exception=util._process_websocket_exception, **options):
+            async for websocket in websockets.asyncio.client.connect(self.url, proxy=self._proxy, process_exception=util._process_websocket_exception, max_size=None, user_agent_header=self.user_agent_header, **options):
                 self.alive = True
                 self._socket_open.set()
                 try:
@@ -177,7 +184,7 @@ class Session(object):
     async def _listen_data_task(self, websocket):
         async for message in websocket:
             await self._eventer.emit("raw", message)
-            # Meshcentral does pong wrong and breaks our parsing, so fix it here.
+            # Meshcentral does pong wrong and breaks our parsing, so fix it here. This is fixed now, but we want compatibility with old versions.
             if message == '{action:"pong"}':
                 message = '{"action":"pong"}'
 
@@ -233,23 +240,28 @@ class Session(object):
     async def __aexit__(self, exc_t, exc_v, exc_tb):
         await self.close()
 
-    @util._check_socket
-    async def _send_command(self, data, name, timeout=None):
-        id = f"meshctrl_{name}_{self._get_command_id()}"
+    def _generate_response_id(self, name):
+        responseid = f"meshctrl_{name}_{self._get_command_id()}"
         # This fixes a very theoretical bug with hash colisions in the case of an infinite int of requests. Now the bug will only happen if there are currently 2**32-1 of the same type of request going out at the same time
-        while id in self._inflight:
-            id = f"meshctrl_{name}_{self._get_command_id()}"
+        while responseid in self._inflight:
+            responseid = f"meshctrl_{name}_{self._get_command_id()}"
+        return responseid
 
-        self._inflight.add(id)
+    @util._check_socket
+    async def _send_command(self, data, name, timeout=None, responseid=None):
+        if responseid is None:
+            responseid = self._generate_response_id(name)
+
+        self._inflight.add(responseid)
         responded = asyncio.Event()
         response = None
         async def _(data):
-            self._inflight.remove(id)
+            self._inflight.remove(responseid)
             nonlocal response
             response = data
             responded.set()
-        self._eventer.once(id, _)
-        await self._message_queue.put(json.dumps(data | {"tag": id, "responseid": id}))
+        self._eventer.once(responseid, _)
+        await self._message_queue.put(json.dumps(data | {"tag": responseid, "responseid": responseid}))
         await asyncio.wait_for(responded.wait(), timeout=timeout)
         if isinstance(response, Exception):
             raise response
@@ -478,10 +490,21 @@ class Session(object):
         if "result" in res0:
             raise exceptions.ServerError(res0["result"])
         if details:
-            nodes = json.loads(res0["data"])
+            nodes = res0["data"]
+            # Accept any number of nested strings, meshcentral is odd
+            while True:
+                try:
+                    nodes = json.loads(nodes)
+                except TypeError:
+                    break
+
             for node in nodes:
                 if node["node"].get("meshid", None):
                     node["node"]["mesh"] = mesh.Mesh(node["node"].get("meshid"), self)
+                if "lastConnect" in node and isinstance(node["lastConnect"], dict):
+                    node["node"]["lastconnect"] = node["lastConnect"].get("time")
+                    node["node"]["lastaddr"] = node["lastConnect"].get("addr")
+                    del node["lastConnect"]
                 details = {}
                 for key, val in node.items():
                     if key != "node":
@@ -533,7 +556,7 @@ class Session(object):
                 data = await event_queue.get()
                 yield data
         finally:
-            self._eventer.off("server_event", _)
+            self._eventer.off("raw", _)
 
     async def events(self, filter=None):
         '''
@@ -1044,6 +1067,30 @@ class Session(object):
             raise exceptions.ServerError(data["result"])
         return True
 
+    async def remove_devices(self, nodeids, timeout=None):
+        '''
+        Remove device(s) from MeshCentral
+
+        Args:
+            nodeids (str|list[str]): nodeid(s) of the device(s) that have to be removed
+            timeout (int): duration in seconds to wait for a response before throwing an error
+        
+        Returns:
+            bool: True on success, raise otherwise
+
+        Raises:
+            :py:class:`~meshctrl.exceptions.ServerError`: Error text from server if there is a failure
+            :py:class:`~meshctrl.exceptions.SocketError`: Info about socket closure
+            asyncio.TimeoutError: Command timed out
+         '''
+        if isinstance(nodeids, str):
+            nodeids = [nodeids]
+
+        data = await self._send_command({ "action": 'removedevices', "nodeids": nodeids}, "remove_devices", timeout=timeout)
+        
+        if data.get("result", "ok").lower() != "ok":
+            raise exceptions.ServerError(data["result"])
+        return True
 
     async def add_device_group(self, name, description="", amtonly=False, features=0, consent=0, timeout=None):
         '''
@@ -1363,10 +1410,10 @@ class Session(object):
                         node["meshid"] = meshid
                         if _mesh is not None:
                             node["mesh"] = _mesh
-                sysinfo["node"] = node
-                sysinfo["nodeid"] = nodeid
-                del sysinfo["result"]
-                del sysinfo["noinfo"]
+                        break
+                else:
+                    continue
+                break
         if node is None:
             raise ValueError("Invalid device id")
         if lastconnect is not None:
@@ -1417,7 +1464,7 @@ class Session(object):
 
     async def run_command(self, nodeids, command, powershell=False, runasuser=False, runasuseronly=False, ignore_output=False, timeout=None):
         '''
-        Run a command on any number of nodes. WARNING: Non namespaced call. Calling this function again before it returns may cause unintended consequences.
+        Run a command on any number of nodes. WARNING: Non namespaced call on older versions of meshcentral (<1.0.22). Calling this function on those versions again before it returns may cause unintended consequences.
 
         Args:
             nodeids (str|list[str]): Unique ids of nodes on which to run the command
@@ -1455,24 +1502,140 @@ class Session(object):
                     return nid
 
         result = {n: {"complete": False, "result": [], "command": command} for n in nodeids}
-        async def _():
+        console_result = {n: {"complete": False, "result": [], "command": command} for n in nodeids}
+        reply_result = {n: {"complete": False, "result": [], "command": command} for n in nodeids}
+        async def _console():
             async for event in self.events({"action": "msg", "type": "console"}):
                 node = match_nodeid(event["nodeid"], nodeids)
                 if node:
                     if event["value"] == "Run commands completed.":
+                        console_result.setdefault(node, {})["complete"] = True
+                        if all(_["complete"] for key, _ in console_result.items()):
+                            break
+                        continue
+                    elif (event["value"].startswith("Run commands")):
+                        continue
+                    console_result[node]["result"].append(event["value"])
+
+        async def _reply(responseid, data=None):
+            # Returns True when all results are in, Falsey otherwise
+            def _parse_event(event):
+                node = match_nodeid(event["nodeid"], nodeids)
+                if node:
+                    reply_result.setdefault(node, {})["complete"] = True
+                    reply_result[node]["result"].append(event["result"])
+                    if all(_["complete"] for key, _ in reply_result.items()):
+                        return True
+            if data is not None:
+                if _parse_event(data):
+                    return
+            async for event in self.events({"action": "msg", "type": "runcommands", "responseid":responseid}):
+                if _parse_event(event):
+                    break
+
+        async def __(command, tg, tasks):
+            nonlocal result
+            responseid = self._generate_response_id("run_command")
+            
+            if not ignore_output:
+                reply_task = tg.create_task(asyncio.wait_for(_reply(responseid), timeout=timeout))
+            # We still need to parse the console results because it sends them without namespace, this will likely break older versions of meshcentral
+            console_task = tg.create_task(asyncio.wait_for(_console(), timeout=timeout))
+            data = await self._send_command(command, "run_command", timeout=timeout, responseid=responseid)
+
+            if data.get("type", None) != "runcommands" and data.get("result", "ok").lower() != "ok":
+                raise exceptions.ServerError(data["result"])
+            elif data.get("type", None) != "runcommands" and data.get("result", "ok").lower() == "ok":
+                expect_response = False
+                if not ignore_output:
+                    reply_task.cancel()
+                    result = console_result
+                userid = (await self.user_info())["_id"]
+                for n in nodeids:
+                    device_info = await self.device_info(n, timeout=timeout)
+                    try:
+                        permissions = device_info.mesh.links.get(userid, {}).get("rights",constants.DeviceRights.norights)
+                        # This should work for device rights, but it only seems to work for mesh rights. Not sure why, but I can't get the events to show up when the user only has individual device rights
+                        # |device_info.get("links", {}).get(userid, {}).get("rights", constants.DeviceRights.norights)
+                        # If we don't have agentconsole rights, we won't be able te read the output, so fill in blanks on this node
+                        if not permissions&constants.DeviceRights.agentconsole:
+                            result[n]["complete"] = True
+                        else:
+                            expect_response = True
+                    except AttributeError:
+                        result[n]["complete"] = True
+                if expect_response:
+                    tasks.append(console_task)
+                else:
+                    console_task.cancel()
+            elif data.get("type", None) == "runcommands":
+                console_task.cancel()
+                if not ignore_output:
+                    result = reply_result
+                    tasks.append(reply_task)
+            else:
+                # if not ignore_output:
+                console_task.cancel()
+                reply_task.cancel()
+                raise exceptions.ServerError(f"Unrecognized response: {data}")
+
+        tasks = []
+        async with asyncio.TaskGroup() as tg:
+            tasks.append(tg.create_task(__({ "action": 'runcommands', "nodeids": nodeids, "type": (2 if powershell else 0), "cmds": command, "runAsUser": runAsUser, "reply": True}, tg, tasks)))
+
+        return {n: v | {"result": "".join(v["result"])} for n,v in result.items()}
+
+    async def run_console_command(self, nodeids, command, powershell=False, runasuser=False, runasuseronly=False, ignore_output=False, timeout=None):
+        '''
+        Run a mesh console command on any number of nodes. WARNING: Non namespaced call. Calling this function again before it returns may cause unintended consequences.
+
+        Args:
+            nodeids (str|list[str]): Unique ids of nodes on which to run the command
+            command (str): Command to run
+            ignore_output (bool): Don't bother trying to get the output. Every device will return an empty string for its result.
+            timeout (int): duration in seconds to wait for a response before throwing an error
+
+        Returns:
+            dict[str, ~meshctrl.types.RunCommandResponse]: Dict containing mapped output of the commands by device
+
+        Raises:
+            :py:class:`~meshctrl.exceptions.ServerError`: Error text from server if there is a failure
+            :py:class:`~meshctrl.exceptions.SocketError`: Info about socket closure
+            ValueError: `Invalid device id` if device is not found
+            asyncio.TimeoutError: Command timed out
+         '''
+        if isinstance(nodeids, str):
+            nodeids = [nodeids]
+
+        def match_nodeid(id, ids):
+            for nid in ids:
+                if (nid == id):
+                    return nid
+                if (nid[6:] == id):
+                    return nid
+                if (f"node//{nid}" == id):
+                    return nid
+
+        result = {n: {"complete": False, "result": [], "command": command} for n in nodeids}
+        async def _console():
+            async for event in self.events({"action": "msg", "type": "console"}):
+                # We can pick up run commands here sometimes if they are run in quick succession. Try to avoid that.
+                if (not event["value"].startswith("Run commands")):
+                    node = match_nodeid(event["nodeid"], nodeids)
+                    if node:
+                        result[node]["result"].append(event["value"])
                         result.setdefault(node, {})["complete"] = True
                         if all(_["complete"] for key, _ in result.items()):
                             break
-                    elif (event["value"].startswith("Run commands")):
-                        continue
-                    result[node]["result"].append(event["value"])
-        async def __(command):
-            data = await self._send_command(command, "run_command", timeout=timeout)
+        async def __(command, tg, tasks):
+            console_task = tg.create_task(asyncio.wait_for(_console(), timeout=timeout))
+            data = await self._send_command(command, "run_console_command", timeout=timeout)
 
-            if data.get("result", "ok").lower() != "ok":
+            if data.get("type", None) != "runcommands" and data.get("result", "ok").lower() != "ok":
                 raise exceptions.ServerError(data["result"])
-        
+            elif data.get("type", None) != "runcommands" and data.get("result", "ok").lower() == "ok":
                 expect_response = False
+                
                 if not ignore_output:
                     userid = (await self.user_info())["_id"]
                     for n in nodeids:
@@ -1488,12 +1651,17 @@ class Session(object):
                                 expect_response = True
                         except AttributeError:
                             result[n]["complete"] = True
+                if expect_response:
+                    tasks.append(console_task)
+                else:
+                    console_task.cancel()
+            else:
+                console_task.cancel()
+                raise exceptions.ServerError(f"Unrecognized response: {data}")
 
         tasks = []
         async with asyncio.TaskGroup() as tg:
-            if expect_response:
-                tasks.append(tg.create_task(asyncio.wait_for(_(), timeout=timeout)))
-            tasks.append(tg.create_task(__({ "action": 'runcommands', "nodeids": nodeids, "type": (2 if powershell else 0), "cmds": command, "runAsUser": runAsUser })))
+            tasks.append(tg.create_task(__({ "action": 'runcommands', "nodeids": nodeids, "type": 4, "cmds": command}, tg, tasks)))
 
         return {n: v | {"result": "".join(v["result"])} for n,v in result.items()}
 
@@ -1736,10 +1904,11 @@ class Session(object):
             tasks.append(tg.create_task(asyncio.wait_for(_(), timeout=timeout)))
             tasks.append({ "action": 'msg', "type": 'openUrl', "nodeid": nodeid, "url": url }, "device_open_url", timeout=timeout)
 
-        res = tasks[1].result()
-        success = tasks[2].result()
         
-        if data.get("result", "ok").lower() != "ok":
+        success = tasks[0].result()
+        res = tasks[1].result()
+
+        if res.get("result", "ok").lower() != "ok":
             raise exceptions.ServerError(data["result"])
 
         if not success:
@@ -1824,7 +1993,7 @@ class Session(object):
 
     async def upload(self, node, source, target, unique_file_tunnel=False, timeout=None):
         '''
-        Upload a stream to a device. This creates an _File and destroys it every call. If you need to upload multiple files, use {@link Session#file_explorer} instead.
+        Upload a stream to a device.
 
         Args:
             node (~meshctrl.device.Device|str): Device or id of device to which to upload the file. If it is a device, it must have a ~meshctrl.mesh.Mesh device associated with it (the default). If it is a string, the device will be fetched prior to tunnel creation.

src/meshctrl/tunnel.py

@@ -52,7 +52,7 @@ class Tunnel(object):
                 options["ssl"] = self._session._ssl_context
 
             if (len(self.node_id.split('/')) != 3):
-                self.node_id = f"node/{self._session._currentDomain or ""}/{self.node_id}"
+                self.node_id = f"node/{self._session._currentDomain or ''}/{self.node_id}"
 
             self._tunnel_id = util._get_random_hex(6)
 
@@ -67,7 +67,7 @@ class Tunnel(object):
             self.url = self._session.url.replace('/control.ashx', '/meshrelay.ashx?browser=1&p=' + str(self._protocol) + '&nodeid=' + self.node_id + '&id=' + self._tunnel_id + '&auth=' + self._authcookie["cookie"])
 
 
-            async for websocket in util.proxy_connect(self.url, proxy_url=self._session._proxy, process_exception=util._process_websocket_exception, **options):
+            async for websocket in websockets.asyncio.client.connect(self.url, proxy=self._session._proxy, process_exception=util._process_websocket_exception, **options):
                 self.alive = True
                 self._socket_open.set()
                 try:

src/meshctrl/util.py

@@ -11,7 +11,6 @@ import ssl
 import functools
 import urllib
 import python_socks
-from python_socks.async_.asyncio import Proxy
 from . import exceptions
 
 def _encode_cookie(o, key):
@@ -140,17 +139,20 @@ def compare_dict(dict1, dict2):
         return False
 
 def _check_socket(f):
-    @functools.wraps(f)
-    async def wrapper(self, *args, **kwargs):
-        try:
-            async with asyncio.TaskGroup() as tg:
-                tg.create_task(asyncio.wait_for(self.initialized.wait(), 10))
-                tg.create_task(asyncio.wait_for(self._socket_open.wait(), 10))
-        finally:
+    async def _check_errs(self):
         if not self.alive and self._main_loop_error is not None:
             raise self._main_loop_error
         elif not self.alive and self.initialized.is_set():
             raise exceptions.SocketError("Socket Closed")
+
+    @functools.wraps(f)
+    async def wrapper(self, *args, **kwargs):
+        try:
+            await asyncio.wait_for(self.initialized.wait(), 10)
+            await _check_errs(self)
+            await asyncio.wait_for(self._socket_open.wait(), 10)
+        finally:
+            await _check_errs(self)
             return await f(self, *args, **kwargs)
     return wrapper
 
@@ -161,17 +163,7 @@ def _process_websocket_exception(exc):
         return exc
     if isinstance(exc, python_socks._errors.ProxyError):
         return None
+    # Proxy errors show up like this now, and it's default to error out. Handle explicitly.
+    if isinstance(exc, websockets.exceptions.InvalidProxyMessage):
+        return None
     return tmp
-
-class proxy_connect(websockets.asyncio.client.connect):
-    def __init__(self,*args, proxy_url=None, **kwargs):
-        self.proxy = None
-        if proxy_url is not None:
-            self.proxy = Proxy.from_url(proxy_url)
-        super().__init__(*args, **kwargs)
-
-    async def create_connection(self, *args, **kwargs):
-        if self.proxy is not None:
-            parsed = urllib.parse.urlparse(self.uri)
-            self.connection_kwargs["sock"] = await self.proxy.connect(dest_host=parsed.hostname, dest_port=parsed.port)
-        return await super().create_connection(*args, **kwargs)

tests/environment/__init__.py

@@ -4,7 +4,10 @@ import subprocess
 import time
 import json
 import atexit
+try:
     import pytest
+except:
+    pass
 import requests
 thisdir = os.path.abspath(os.path.dirname(__file__))
 
@@ -37,7 +40,9 @@ class Agent(object):
         self._clienturl = clienturl
         self._dockerurl = dockerurl
         r = requests.post(f"{self._clienturl}/add-agent", json={"url": f"{self._dockerurl}", "meshid": self.meshid})
-        self.nodeid = r.json()["id"]
+        agent_json = r.json()
+        self.nodeid = agent_json["id"]
+        self.nodehex = agent_json["hex"]
 
     def __enter__(self):
         return self
@@ -62,10 +67,13 @@ class TestEnvironment(object):
             return self
         # Destroy the env in case it wasn't killed correctly last time.
         subprocess.check_call(["docker", "compose", "down"], stdout=subprocess.DEVNULL, cwd=thisdir)
-        self._subp = _docker_process = subprocess.Popen(["docker", "compose", "up", "--build", "--force-recreate", "--no-deps"], stdout=subprocess.DEVNULL, cwd=thisdir)
+        self._subp = _docker_process = subprocess.Popen(["docker", "compose", "up", "--build", "--force-recreate", "--no-deps"], cwd=thisdir)
         if not self._wait_for_meshcentral():
             self.__exit__(None, None, None)
             raise Exception("Failed to create docker instance")
+        if not self._wait_for_client_server():
+            self.__exit__(None, None, None)
+            raise Exception("Failed to create client server")
         return self
 
     def _wait_for_meshcentral(self, timeout=30):
@@ -88,6 +96,26 @@ class TestEnvironment(object):
             return False
         return True
 
+    def _wait_for_client_server(self, timeout=30):
+        start = time.time()
+        while time.time() - start < timeout:
+            try:
+                data = subprocess.check_output(["docker", "inspect", "meshctrl-client", "--format='{{json .State.Health}}'"], cwd=thisdir, stderr=subprocess.DEVNULL)
+                # docker outputs for humans, not computers. This is the easiest way to chop off the ends
+                data = json.loads(data.strip()[1:-1])
+            except Exception as e:
+                time.sleep(1)
+                continue
+            try:
+                if data["Status"] == "healthy":
+                    break
+            except:
+                pass
+            time.sleep(1)
+        else:
+            return False
+        return True
+
     def __exit__(self, exc_t, exc_v, exc_tb):
         pass
 
@@ -110,10 +138,13 @@ def _kill_docker_process():
 
 atexit.register(_kill_docker_process)
 
+try:
     @pytest.fixture(scope="session")
     def env():
         with TestEnvironment() as e:
             yield e
+except:
+    pass
 
 
 if __name__ == "__main__":

tests/environment/client.dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12
+FROM python:3.13
 WORKDIR /usr/local/app
 
 # Install the application dependencies

tests/environment/compose.yaml

@@ -9,6 +9,8 @@ services:
     image: client
     build:
       dockerfile: client.dockerfile
+    sysctls:
+      net.ipv6.conf.all.disable_ipv6: 1
     ports:
       - 5000:5000
     depends_on:
@@ -20,6 +22,10 @@ services:
     #   - ./meshcentral/mongodb_data:/data/db
     networks:
       - meshctrl
+    healthcheck:
+      test: curl --fail http://localhost:5000/ || exit 1
+      interval: 5s
+      timeout: 120s
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -28,6 +34,8 @@ services:
     container_name: meshctrl-meshcentral
     # use the official meshcentral container
     image: meshcentral
+    sysctls:
+      net.ipv6.conf.all.disable_ipv6: 1
     build:
       dockerfile: meshcentral.dockerfile
     ports:
@@ -55,6 +63,8 @@ services:
     image: ubuntu/squid:latest
     restart: unless-stopped
     container_name: meshctrl-squid
+    sysctls:
+      net.ipv6.conf.all.disable_ipv6: 1
     ports:
       - 3128:3128
 

tests/environment/config/meshcentral/overrides/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

tests/environment/config/squid/conf.d/meshctrl.conf

@@ -1,9 +1,19 @@
 # Logs are managed by logrotate on Debian
 logfile_rotate 0
 
-acl all src all
+acl to_ipv6 dst ipv6
+acl from_ipv6 src ipv6
+
+acl to_ipv4 dst ipv4
+acl from_ipv4 src ipv4
+
+#acl all src all
 acl Safe_ports port 8086
 acl SSS_ports port 8086
+http_access allow to_ipv4
+http_access allow from_ipv4
+http_access deny to_ipv6
+http_access deny from_ipv6
 http_access allow all
 debug_options ALL,0 85,2 88,2
 

tests/environment/meshcentral.dockerfile

@@ -1,7 +1,8 @@
-FROM ghcr.io/ylianst/meshcentral:latest
+FROM ghcr.io/ylianst/meshcentral:1.1.56
 RUN apk add curl
 RUN apk add python3
 WORKDIR /opt/meshcentral/
 COPY ./scripts/meshcentral ./scripts
 COPY ./config/meshcentral/data /opt/meshcentral/meshcentral-data
-CMD ["python3", "/opt/meshcentral/scripts/create_users.py"]
+COPY ./config/meshcentral/overrides /opt/meshcentral/meshcentral
+ENTRYPOINT ["python3", "/opt/meshcentral/scripts/create_users.py"]

tests/environment/scripts/client/agent_server.py

@@ -53,7 +53,7 @@ def add_agent():
         time.sleep(.1)
     else:
         raise Exception(f"Failed to start agent: {text}")
-    return {"id": agent_id}
+    return {"id": agent_id, "hex": agent_hex}
 
 @api.route('/remove-agent/<agentid>', methods=['POST'])
 def remove_agent(agentid):
@@ -62,7 +62,7 @@ def remove_agent(agentid):
 
 @api.route('/', methods=['GET'])
 def slash():
-    return [_["id"] for _ in agents]
+    return [value["id"] for key, value in agents.items()]
 
 if __name__ == '__main__':
     api.run()

tests/environment/scripts/meshcentral/create_users.py

@@ -7,9 +7,9 @@ thisdir = os.path.abspath(os.path.dirname(__file__))
 with open(os.path.join(thisdir, "users.json")) as infile:
     users = json.load(infile)
 for username, password in users.items():
-    subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--createaccount", username, "--pass", password, "--name", username])
+    print(subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--createaccount", username, "--pass", password, "--name", username]))
 
 
-subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--adminaccount", "admin"])
+print(subprocess.check_output(["node", "/opt/meshcentral/meshcentral", "--adminaccount", "admin"]))
 
-subprocess.call(["bash", "/opt/meshcentral/startup.sh"])
+subprocess.call(["bash", "/opt/meshcentral/entrypoint.sh"])

tests/requirements.txt

@@ -1,6 +1,6 @@
 requests
 pytest-asyncio
 cffi==1.17.1
-cryptography==43.0.3
 pycparser==2.22
-websockets==13.1
+cryptography~=46.0.5
+websockets~=16.0.0

tests/test_files.py

@@ -120,7 +120,7 @@ async def test_upload_download(env):
                     downfilestream.seek(0)
 
                     start = time.perf_counter()
-                    r = await files.download(f"{pwd}/test", downfilestream, skip_http_attempt=True, timeout=5)
+                    r = await files.download(f"{pwd}/test", downfilestream, skip_http_attempt=True, timeout=20)
                     print("\ninfo files_download: {}\n".format(r))
                     assert r["result"] == True, "Download failed"
                     assert r["size"] == len(randdata), "Downloaded wrong number of bytes"

tests/test_sanity.py

@@ -9,16 +9,7 @@ import requests
 
 async def test_sanity(env):
     async with meshctrl.Session(env.mcurl, user="unprivileged", password=env.users["unprivileged"], ignore_ssl=True) as s:
-        got_pong = asyncio.Event()
-        async def _():
-            async for raw in s.raw_messages():
-                if raw == '{action:"pong"}':
-                    got_pong.set()
-                    break
-        ping_task = None
         async with asyncio.TaskGroup() as tg:
-            tg.create_task(asyncio.wait_for(_(), timeout=5))
-            tg.create_task(asyncio.wait_for(got_pong.wait(), timeout=5))
             ping_task = tg.create_task(s.ping(timeout=10))
         print("\ninfo ping: {}\n".format(ping_task.result()))
         print("\ninfo user_info: {}\n".format(await s.user_info()))

tests/test_session.py

@@ -5,6 +5,8 @@ import meshctrl
 import requests
 import random
 import io
+import traceback
+import time
 thisdir = os.path.dirname(os.path.realpath(__file__))
 
 async def test_admin(env):
@@ -44,8 +46,10 @@ async def test_auto_reconnect(env):
         for i in range(3):
             try:
                 await admin_session.ping(timeout=10)
-            except:
-                continue
+            except* Exception as e:
+                print("".join(traceback.format_exception(e)))
+                pass
+            else:
                 break
         else:
             raise Exception("Failed to reconnect")
@@ -55,6 +59,7 @@ async def test_auto_reconnect(env):
             try:
                 await admin_session.ping(timeout=10)
             except* Exception as e:
+                print("".join(traceback.format_exception(e)))
                 pass
             else:
                 break
@@ -77,6 +82,17 @@ async def test_users(env):
         pass
     else:
         raise Exception("Connected with no password")
+
+    start = time.time()
+    try:
+        async with meshctrl.Session(env.mcurl, user="admin", password="The wrong password", ignore_ssl=True) as admin_session:
+            pass
+    except* meshctrl.exceptions.ServerError as eg:
+        assert str(eg.exceptions[0]) == "Invalid Auth" or eg.exceptions[0].message == "Invalid Auth", "Didn't get invalid auth message"
+        assert time.time() - start < 10, "Invalid auth wasn't raised until after timeout"
+        pass
+    else:
+        raise Exception("Connected with bad password")
     async with meshctrl.Session(env.mcurl+"/", user="admin", password=env.users["admin"], ignore_ssl=True) as admin_session,\
                meshctrl.Session(env.mcurl, user="privileged", password=env.users["privileged"], ignore_ssl=True) as privileged_session,\
                meshctrl.Session(env.mcurl, user="unprivileged", password=env.users["unprivileged"], ignore_ssl=True) as unprivileged_session:
@@ -187,21 +203,24 @@ async def test_mesh_device(env):
 
         assert r[0].description == "New description", "Description either failed to change, or was changed by a user without permission to do so"
 
-        with env.create_agent(mesh.short_meshid) as agent:
+        # There once was a bug that occured whenever running run_commands with multiple meshes. We need to add devices to both meshes to be sure that bug is squashed.
+        with env.create_agent(mesh.short_meshid) as agent,\
+             env.create_agent(mesh.short_meshid) as agent2,\
+             env.create_agent(mesh2.short_meshid) as agent3:
             # Test agent added to device group being propagated correctly
             # Create agent isn't so good at waiting for the agent to show in the sessions. Give it a couple seconds to appear.
             for i in range(3):
                 try:
                     r = await admin_session.list_devices(timeout=10)
                     print("\ninfo list_devices: {}\n".format(r))
-                    assert len(r) == 1, "Incorrect number of agents connected"
+                    assert len(r) == 3, "Incorrect number of agents connected"
                 except:
                     if i == 2:
                         raise
                     await asyncio.sleep(1)
                 else:
                     break
-            assert len(await privileged_session.list_devices(timeout=10)) == 1, "Incorrect number of agents connected"
+            assert len(await privileged_session.list_devices(timeout=10)) == 2, "Incorrect number of agents connected"
             assert len(await unprivileged_session.list_devices(timeout=10)) == 0, "Unprivileged account has access to agent it should not"
 
             r = await admin_session.list_devices(details=True, timeout=10)
@@ -213,6 +232,9 @@ async def test_mesh_device(env):
             r = await admin_session.list_devices(meshid=mesh.meshid, timeout=10)
             print("\ninfo list_devices_meshid: {}\n".format(r))
 
+            r = await admin_session.device_info(agent.nodeid, timeout=10)
+            print("\ninfo admin_device_info: {}\n".format(r))
+
             # Test editing device info propagating correctly
             assert await admin_session.edit_device(agent.nodeid, name="new_name", description="New Description", tags="device", consent=meshctrl.constants.ConsentFlags.all, timeout=10), "Failed to edit device info"
 
@@ -221,12 +243,49 @@ async def test_mesh_device(env):
             assert await admin_session.edit_device(agent.nodeid, consent=meshctrl.constants.ConsentFlags.none, timeout=10), "Failed to edit device info"
 
             # Test run_commands
-            r = await admin_session.run_command(agent.nodeid, "ls", timeout=10)
+            r = await admin_session.run_command([agent.nodeid, agent2.nodeid], "ls", timeout=10)
             print("\ninfo run_command: {}\n".format(r))
             assert "meshagent" in r[agent.nodeid]["result"], "ls gave incorrect data"
+            assert "meshagent" in r[agent2.nodeid]["result"], "ls gave incorrect data"
+            assert "Run commands completed." not in r[agent.nodeid]["result"], "Didn't parse run command ending correctly"
+            assert "Run commands completed." not in r[agent2.nodeid]["result"], "Didn't parse run command ending correctly"
             assert "meshagent" in (await privileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
 
-            # Test run commands with ndividual device permissions
+            # Test run_commands ignore output
+            r = await admin_session.run_command([agent.nodeid, agent2.nodeid], "ls", ignore_output=True, timeout=10)
+            print("\ninfo run_command ignore_output: {}\n".format(r))
+            assert r[agent.nodeid]["result"] == '', "Ignore output returned an output"
+            assert r[agent2.nodeid]["result"] == '', "Ignore output returned an output"
+
+            # Test run_commands missing device
+            try:
+                await admin_session.run_command([agent.nodeid, "notanid"], "ls", timeout=10)
+            except* (meshctrl.exceptions.ServerError, ValueError):
+                pass
+            else:
+                raise Exception("Run command on a device that doesn't exist did not raise an exception")
+
+            # Test run_console_command
+            r = await admin_session.run_console_command([agent.nodeid, agent2.nodeid], "info", timeout=10)
+            print("\ninfo run_console_command: {}\n".format(r))
+            assert agent.nodeid in r[agent.nodeid]["result"], "Run console command gave bad response"
+            assert agent2.nodeid in r[agent2.nodeid]["result"], "Run console command gave bad response"
+
+            # Test run_console_command ignore output
+            r = await admin_session.run_console_command([agent.nodeid, agent2.nodeid], "info", timeout=10, ignore_output=True)
+            print("\ninfo run_console_command ignore_output: {}\n".format(r))
+            assert r[agent.nodeid]["result"] == '', "Ignore output returned an output"
+            assert r[agent2.nodeid]["result"] == '', "Ignore output returned an output"
+
+            # Test run_commands missing device
+            try:
+                await admin_session.run_console_command([agent.nodeid, "notanid"], "info", timeout=10)
+            except* (meshctrl.exceptions.ServerError, ValueError):
+                pass
+            else:
+                raise Exception("Run console command on a device that doesn't exist did not raise an exception")
+
+            # Test run commands with individual device permissions
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)
             except* (meshctrl.exceptions.ServerError, ValueError):
@@ -241,7 +300,7 @@ async def test_mesh_device(env):
             else:
                 raise Exception("Unprivileged user has access to device it should not")
 
-            assert (await admin_session.add_users_to_device((await unprivileged_session.user_info())["_id"], agent.nodeid, meshctrl.constants.MeshRights.norights)), "Failed to add user to device"
+            assert (await admin_session.add_users_to_device((await unprivileged_session.user_info())["_id"], agent.nodeid, meshctrl.constants.DeviceRights.norights)), "Failed to add user to device"
 
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", ignore_output=True, timeout=10)
@@ -252,19 +311,21 @@ async def test_mesh_device(env):
 
             # Test getting individual device info
             r = await unprivileged_session.device_info(agent.nodeid, timeout=10)
-            print("\ninfo device_info: {}\n".format(r))
+            print("\ninfo unprivileged_device_info: {}\n".format(r))
 
             # This device info includes the mesh ID of the device, even though the user doesn't have acces to that mesh. That's odd.
             # assert r.meshid is None, "Individual device is exposing its meshid"
 
             assert r.links[(await unprivileged_session.user_info())["_id"]]["rights"] == meshctrl.constants.DeviceRights.norights, "Unprivileged user has too many rights!"
 
-            assert (await admin_session.add_users_to_device([(await unprivileged_session.user_info())["_id"]], agent.nodeid, meshctrl.constants.DeviceRights.remotecontrol|meshctrl.constants.DeviceRights.agentconsole|meshctrl.constants.DeviceRights.remotecommands)), "Failed to modify user's permissions"
+            assert (await admin_session.add_users_to_device([(await unprivileged_session.user_info())["_id"]], agent.nodeid, meshctrl.constants.DeviceRights.fullrights)), "Failed to modify user's permissions"
 
-            assert (await unprivileged_session.device_info(agent.nodeid, timeout=10)).links[(await unprivileged_session.user_info())["_id"]]["rights"] == meshctrl.constants.DeviceRights.remotecontrol|meshctrl.constants.DeviceRights.agentconsole|meshctrl.constants.DeviceRights.remotecommands, "Adding permissions did not update unprivileged user."
+            assert (await unprivileged_session.device_info(agent.nodeid, timeout=10)).links[(await unprivileged_session.user_info())["_id"]]["rights"] == meshctrl.constants.DeviceRights.fullrights, "Adding permissions did not update unprivileged user."
 
-            # For now, this expects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
+            # For now, this expects no response. If we ever figure out why the server isn't sending console information to us when it should, fix this.
             # assert "meshagent" in (await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
+            # Meshcentral has a 10 second cache on user perms.
+            #await asyncio.sleep(15)
             await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)
 
             assert await admin_session.move_to_device_group(agent.nodeid, mesh2.meshid, timeout=5), "Failed to move mesh to new device group"
@@ -278,7 +339,7 @@ async def test_mesh_device(env):
 
             assert await admin_session.move_to_device_group([agent.nodeid], mesh.name, isname=True, timeout=5), "Failed to move mesh to new device group by name"
 
-            # For now, this expe namects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
+            # For now, this expects no response. If we ever figure out why the server isn't sending console information te us when it should, fix this.
             # assert "meshagent" in (await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10))[agent.nodeid]["result"], "ls gave incorrect data"
             try:
                 await unprivileged_session.run_command(agent.nodeid, "ls", timeout=10)
@@ -288,6 +349,15 @@ async def test_mesh_device(env):
             r = await admin_session.remove_users_from_device_group((await privileged_session.user_info())["_id"], mesh.meshid, timeout=10)
             print("\ninfo remove_users_from_device_group: {}\n".format(r))
             assert (r[(await privileged_session.user_info())["_id"]]["success"]), "Failed to remove user from device group"
+
+            await admin_session.remove_devices(agent2.nodeid, timeout=10)
+            try:
+                await admin_session.device_info(agent2.nodeid, timeout=10)
+            except ValueError:
+                pass
+            else:
+                raise Exception("Device not deleted")
+
             assert (await admin_session.remove_users_from_device(agent.nodeid, (await unprivileged_session.user_info())["_id"], timeout=10)), "Failed to remove user from device"