_import3 no longer blocks event loop during decryption

Now running encrytion of accounts in ProcessPoolExecutor
This commit is contained in:
StNicolay 2023-01-05 13:48:23 +03:00
parent f4a5f51b23
commit 157c2c4aa2

View File

@ -1,6 +1,7 @@
import asyncio import asyncio
import functools import functools
import gc import gc
import itertools
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
import telebot import telebot
@ -639,14 +640,26 @@ async def _import3(
# List of names of accounts, which failed to be added to the database # List of names of accounts, which failed to be added to the database
# or failed the tests # or failed the tests
failed: list[str] = [] failed: list[str] = []
tasks: list[asyncio.Future[db.models.Account]] = []
loop = asyncio.get_running_loop()
with ProcessPoolExecutor() as pool:
for account in accounts: for account in accounts:
if not check_account(account): if not check_account(account):
failed.append(account.name) failed.append(account.name)
continue continue
account = encryption.accounts.encrypt(account, text) function = functools.partial(
result = db.add.add_account(engine, account) encryption.accounts.encrypt,
if not result: account,
failed.append(account.name) text,
)
tasks.append(loop.run_in_executor(pool, function))
enc_accounts: list[db.models.Account] = await asyncio.gather(*tasks)
results = db.add.add_accounts(engine, enc_accounts)
failed_accounts = itertools.compress(
enc_accounts, (not result for result in results)
)
failed.extend((account.name for account in failed_accounts))
if failed: if failed:
await send_deleteable_message( await send_deleteable_message(
@ -655,7 +668,7 @@ async def _import3(
else: else:
await send_tmp_message(bot, mes.chat.id, "Успех") await send_tmp_message(bot, mes.chat.id, "Успех")
del text, mes, accounts del text, mes, accounts, function, tasks, failed_accounts
gc.collect() gc.collect()