Merge pull request #3642 from lbryio/libtorrent

use official libtorrent, fix tests, make it a normal dependency

Author: shyba

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
           key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
           restore-keys: ${{ runner.os }}-pip-
       - run: pip install --user --upgrade pip wheel
-      - run: pip install -e .[torrent,lint]
+      - run: pip install -e .[lint]
       - run: make lint
 
   tests-unit:
@@ -50,7 +50,7 @@ jobs:
           string: ${{ runner.os }}
       - run: python -m pip install --user --upgrade pip wheel
       - if: startsWith(runner.os, 'linux')
-        run: pip install -e .[torrent,test]
+        run: pip install -e .[test]
       - if: startsWith(runner.os, 'linux')
         env:
           HOME: /tmp

lbry/extras/daemon/components.py

@@ -28,11 +28,7 @@
 from lbry.wallet import WalletManager
 from lbry.wallet.usage_payment import WalletServerPayer
 from lbry.torrent.tracker import TrackerClient
-
-try:
-    from lbry.torrent.session import TorrentSession
-except ImportError:
-    TorrentSession = None
+from lbry.torrent.session import TorrentSession
 
 log = logging.getLogger(__name__)
 
@@ -361,10 +357,6 @@ async def start(self):
         wallet = self.component_manager.get_component(WALLET_COMPONENT)
         node = self.component_manager.get_component(DHT_COMPONENT) \
             if self.component_manager.has_component(DHT_COMPONENT) else None
-        try:
-            torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT) if TorrentSession else None
-        except NameError:
-            torrent = None
         log.info('Starting the file manager')
         loop = asyncio.get_event_loop()
         self.file_manager = FileManager(
@@ -373,7 +365,8 @@ async def start(self):
         self.file_manager.source_managers['stream'] = StreamManager(
             loop, self.conf, blob_manager, wallet, storage, node,
         )
-        if TorrentSession and LIBTORRENT_COMPONENT not in self.conf.components_to_skip:
+        if self.component_manager.has_component(LIBTORRENT_COMPONENT):
+            torrent = self.component_manager.get_component(LIBTORRENT_COMPONENT)
             self.file_manager.source_managers['torrent'] = TorrentManager(
                 loop, self.conf, torrent, storage, self.component_manager.analytics_manager
             )
@@ -502,9 +495,8 @@ async def get_status(self):
         }
 
     async def start(self):
-        if TorrentSession:
-            self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
-            await self.torrent_session.bind()  # TODO: specify host/port
+        self.torrent_session = TorrentSession(asyncio.get_event_loop(), None)
+        await self.torrent_session.bind()  # TODO: specify host/port
 
     async def stop(self):
         if self.torrent_session:

lbry/torrent/session.py

@@ -10,47 +10,13 @@
 import libtorrent
 
 
-NOTIFICATION_MASKS = [
-    "error",
-    "peer",
-    "port_mapping",
-    "storage",
-    "tracker",
-    "debug",
-    "status",
-    "progress",
-    "ip_block",
-    "dht",
-    "stats",
-    "session_log",
-    "torrent_log",
-    "peer_log",
-    "incoming_request",
-    "dht_log",
-    "dht_operation",
-    "port_mapping_log",
-    "picker_log",
-    "file_progress",
-    "piece_progress",
-    "upload",
-    "block_progress"
-]
 log = logging.getLogger(__name__)
-
-
 DEFAULT_FLAGS = (  # fixme: somehow the logic here is inverted?
         libtorrent.add_torrent_params_flags_t.flag_auto_managed
         | libtorrent.add_torrent_params_flags_t.flag_update_subscribe
 )
 
 
-def get_notification_type(notification) -> str:
-    for i, notification_type in enumerate(NOTIFICATION_MASKS):
-        if (1 << i) & notification:
-            return notification_type
-    raise ValueError("unrecognized notification type")
-
-
 class TorrentHandle:
     def __init__(self, loop, executor, handle):
         self._loop = loop
@@ -156,10 +122,8 @@ async def add_fake_torrent(self):
     async def bind(self, interface: str = '0.0.0.0', port: int = 10889):
         settings = {
             'listen_interfaces': f"{interface}:{port}",
-            'enable_outgoing_utp': True,
-            'enable_incoming_utp': True,
-            'enable_outgoing_tcp': False,
-            'enable_incoming_tcp': False
+            'enable_natpmp': False,
+            'enable_upnp': False
         }
         self._session = await self._loop.run_in_executor(
             self._executor, libtorrent.session, settings  # pylint: disable=c-extension-no-member

scripts/hook-libtorrent.py

@@ -0,0 +1,24 @@
+"""
+Hook for libtorrent.
+"""
+
+import os
+import glob
+import os.path
+from PyInstaller.utils.hooks import get_module_file_attribute
+from PyInstaller import compat
+
+
+def get_binaries():
+    if compat.is_win:
+        files = ('c:/Windows/System32/libssl-1_1-x64.dll', 'c:/Windows/System32/libcrypto-1_1-x64.dll')
+        for file in files:
+            if not os.path.isfile(file):
+                print(f"MISSING {file}")
+        return [(file, '.') for file in files]
+    return []
+
+
+binaries = get_binaries()
+for file in glob.glob(os.path.join(get_module_file_attribute('libtorrent'), 'libtorrent*pyd*')):
+    binaries.append((file, 'libtorrent'))

setup.py

@@ -46,9 +46,9 @@
         'coincurve==15.0.0',
         'pbkdf2==1.3',
         'filetype==1.0.9',
+        'libtorrent==2.0.6',
     ],
     extras_require={
-        'torrent': ['lbry-libtorrent'],
         'lint': [
             'pylint==2.10.0'
         ],

tests/integration/datanetwork/test_file_commands.py

@@ -18,11 +18,17 @@ def __init__(self, *a, **kw):
         super().__init__(*a, **kw)
         self.skip_libtorrent = False
 
+    async def add_forever(self):
+        while True:
+            for handle in self.client_session._handles.values():
+                handle._handle.connect_peer(('127.0.0.1', 4040))
+            await asyncio.sleep(.1)
+
     async def initialize_torrent(self, tx_to_update=None):
         if not hasattr(self, 'seeder_session'):
             self.seeder_session = TorrentSession(self.loop, None)
             self.addCleanup(self.seeder_session.stop)
-            await self.seeder_session.bind(port=4040)
+            await self.seeder_session.bind('127.0.0.1', port=4040)
         btih = await self.seeder_session.add_fake_torrent()
         address = await self.account.receiving.get_or_create_usable_address()
         if not tx_to_update:
@@ -40,8 +46,9 @@ async def initialize_torrent(self, tx_to_update=None):
         await tx.sign([self.account])
         await self.broadcast_and_confirm(tx)
         self.client_session = self.daemon.file_manager.source_managers['torrent'].torrent_session
-        self.client_session._session.add_dht_node(('localhost', 4040))
         self.client_session.wait_start = False  # fixme: this is super slow on tests
+        task = asyncio.create_task(self.add_forever())
+        self.addCleanup(task.cancel)
         return tx, btih
 
     @skipIf(TorrentSession is None, "libtorrent not installed")

tox.ini

@@ -5,7 +5,6 @@ deps =
 extras =
   test
   hub
-  torrent
 changedir = {toxinidir}/tests
 setenv =
   HOME=/tmp