Skip to content

updated #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tameen wants to merge 1 commit into
base: kashifpk/convert-to-poetry-fastapi-and-typer
Choose a base branch
from
Open

updated #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry.scripts]
quranref = 'quranref.cli:main'
quranref_populate = 'quranref.scripts.populate:main'
# quranref_populate = 'quranref.scripts.populate:main'
# quranref_import_surah_info = 'quranref.scripts.surah_info_import:main'
# quranref_import_text = 'quranref.scripts.text_import:main'
# quranref_make_words = 'quranref.scripts.make_words:main'
Expand Down
85 changes: 40 additions & 45 deletions quranref/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,12 @@
import typer
from time import sleep
from .graph_models import get_gdb
from .graph_models.quran_graph import QuranGraph
from .graph_models.quran_graph import Surah
from .lib.surah_info import surah_info
from .graph_models.quran_graph import Aya, AyaText
import time
from quranref.graph_models import get_gdb
from quranref.graph_models.quran_graph import QuranGraph
from quranref.graph_models.quran_graph import Surah
from quranref.lib.surah_info import surah_info
from quranref.graph_models.quran_graph import Aya, AyaText

cmd_app = typer.Typer()
"""
def iteration():
for i in range(1000):
yield i
"""

@cmd_app.command()
def populate():
Expand All @@ -23,6 +18,8 @@ def populate():

@cmd_app.command()
def import_surahs():
"""Importing Surahs."""
gdb = get_gdb()
surah_number = 1
for s_info in surah_info:
s = Surah(
Expand All @@ -36,48 +33,46 @@ def import_surahs():
rukus=s_info['rukus'],
total_ayas=s_info['total_ayas']
)

gdb = get_gdb()
gdb.add(s)
surah_number += 1


@cmd_app.command()
def import_text(language: str, text_name: str, filename: str):
"""
with typer.progressbar(iteration(), length=1000, label="Progressing") as progress:
for i in progress:
sleep(0.01)
"""
file = open(filename)

"""Importing Text"""
file = open('data/'+filename)
bismillah_text = ''
current_surah = ''

for line in file:
if not line.strip():
break

surah, aya, content = line.split('|')

if not bismillah_text:
bismillah_text = content.strip()

if current_surah != surah and content.startswith(bismillah_text):
# new surah starting, separate bismillah and save as aya 0 for surah
content = content[len(bismillah_text):].strip()
if content:
aya_doc = Aya.new(surah_number=surah, aya_number=0)
else:
aya_doc = Aya.new(surah_number=surah, aya_number=aya)

AyaText.new(aya_doc, bismillah_text, language, text_name)
current_surah = surah

if content:
aya_doc = Aya.new(surah_number=surah, aya_number=aya)
AyaText.new(aya_doc, content.strip(), language, text_name)

total = 0
file_range = len(file.readlines())
Copy link
Copy Markdown
Owner

@kashifpk kashifpk Jun 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In order to introduce the progress bar, the logic now reads the entire file at the start. The previous logic was meant so even large files don't require big memory overhead because of the program loading the full file. In previous implementation it was just one line at a time that was being read from the file.

Copy link
Copy Markdown
Owner

@kashifpk kashifpk Jun 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

May be to keep the progress bar and avoid the double for loop below we could do something like:

file_lines = file.readlines()
total_lines = len(file_lines)
with typer.progressbar(range(total_lines)) as progress:
    for value in progress:
        line = file_lines[value]

with typer.progressbar(range(file_range)) as progress:
for value in progress:
for line in file:
if not line.strip():
break

surah, aya, content = line.split('|')

if not bismillah_text:
bismillah_text = content.strip()

if current_surah != surah and content.startswith(bismillah_text):
# new surah starting, separate bismillah and save as aya 0 for surah
content = content[len(bismillah_text):].strip()
if content:
aya_doc = Aya.new(surah_number=surah, aya_number=0)
else:
aya_doc = Aya.new(surah_number=surah, aya_number=aya)

AyaText.new(aya_doc, bismillah_text, language, text_name)
current_surah = surah

if content:
aya_doc = Aya.new(surah_number=surah, aya_number=aya)
AyaText.new(aya_doc, content.strip(), language, text_name)
time.sleep(0.01)
total += 1
print(f"Processed {total} things.")
@cmd_app.command()
def test():
"""A test command."""
Expand Down