diff --git a/examples/books.py b/examples/books.py
index 92e39102..895f3eb2 100644
--- a/examples/books.py
+++ b/examples/books.py
@@ -52,10 +52,10 @@ class Application:
index: yields rows for embeddings indexing if True, otherwise yields database rows
"""
- with open(os.path.join(self.path, "filtered.txt")) as infile:
+ with open(os.path.join(self.path, "filtered.txt"), encoding="utf-8") as infile:
for x, row in enumerate(infile):
if x % 1000 == 0:
- print("Processed %d rows" % x, end="\r")
+ print(f"Processed {x} rows", end="\r")
row = row.split("\t")
uid, data = row[1], json.loads(row[4])
@@ -68,7 +68,7 @@ class Application:
if index:
yield (uid, data["title"] + ". " + description, None)
else:
- cover = "%d" % data["covers"][0] if "covers" in data and data["covers"] else None
+ cover = f"{data['covers'][0]}" if "covers" in data and data["covers"] else None
yield (uid, data["title"], description, cover)
def database(self):
@@ -155,14 +155,14 @@ class Application:
if result:
# Build cover image
cover = (
- "
" % result[2]
+ f"
"
if result[2]
else "
"
)
# Append book link
- cover = "%s" % (uid, cover)
- title = "%s" % (uid, result[0])
+ cover = f"{cover}"
+ title = f"{result[0]}"
results.append({"Cover": cover, "Title": title, "Description": result[1]})
diff --git a/examples/workflows.py b/examples/workflows.py
index e0dc64ec..c894a459 100644
--- a/examples/workflows.py
+++ b/examples/workflows.py
@@ -480,7 +480,7 @@ class Application:
# Generate workflow file
workflow = os.path.join(tempfile.gettempdir(), "workflow.yml")
- with open(workflow, "w") as f:
+ with open(workflow, "w", encoding="utf-8") as f:
f.write(config)
os.environ["CONFIG"] = workflow
@@ -509,7 +509,7 @@ class Application:
text = [text for uid, text, _ in self.data if uid == key][0]
if key and key.lower().startswith("http"):
- return "%s" % (key, text)
+ return f"{text}"
return text
@@ -577,7 +577,7 @@ class Application:
)
if query:
- df = pd.DataFrame([{"content": self.find(uid), "score": "%.2f" % score} for uid, score in self.embeddings.search(query, limit)])
+ df = pd.DataFrame([{"content": self.find(uid), "score": f"{score:.2}"} for uid, score in self.embeddings.search(query, limit)])
st.write(df.to_html(escape=False), unsafe_allow_html=True)
def parse(self, data):
@@ -631,7 +631,7 @@ class Application:
api = col2.button("API", help="Start an API instance within this application")
if api:
- with st.spinner("Running workflow '%s' via API service, click stop to terminate" % name):
+ with st.spinner(f"Running workflow '{name}' via API service, click stop to terminate"):
self.api(config)
col3.download_button("Export", config, file_name="workflow.yml", help="Export the API workflow as YAML")
diff --git a/src/python/txtai/api/application.py b/src/python/txtai/api/application.py
index 75ab35e9..a8cdbcfc 100644
--- a/src/python/txtai/api/application.py
+++ b/src/python/txtai/api/application.py
@@ -29,8 +29,6 @@ def get():
API instance
"""
- # pylint: disable=W0603
- global INSTANCE
return INSTANCE
@@ -44,7 +42,7 @@ def start():
global INSTANCE
# Load YAML settings
- with open(os.getenv("CONFIG"), "r") as f:
+ with open(os.getenv("CONFIG"), "r", encoding="utf-8") as f:
# Read configuration
config = yaml.safe_load(f)
diff --git a/src/python/txtai/api/cluster.py b/src/python/txtai/api/cluster.py
index 745c59e0..74d9332b 100644
--- a/src/python/txtai/api/cluster.py
+++ b/src/python/txtai/api/cluster.py
@@ -45,9 +45,9 @@ class Cluster:
"""
# Build URL
- action = "search?query=%s" % query
+ action = f"search?query={query}"
if limit:
- action += "&limit=%d" % limit
+ action += f"&limit={limit}"
# Run query and flatten results into single results list
results = []
@@ -174,7 +174,7 @@ class Cluster:
"""
# Get urls
- urls = ["%s/%s" % (shard, action) for shard in self.shards]
+ urls = [f"{shard}/{action}" for shard in self.shards]
close = False
# Use existing loop if available, otherwise create one
diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.py
index d0f5b5ee..24897b8a 100644
--- a/src/python/txtai/embeddings/base.py
+++ b/src/python/txtai/embeddings/base.py
@@ -303,7 +303,7 @@ class Embeddings:
"""
# Index configuration
- with open("%s/config" % path, "rb") as handle:
+ with open(f"{path}/config", "rb") as handle:
self.config = pickle.load(handle)
# Build full path to embedding vectors file
@@ -312,11 +312,11 @@ class Embeddings:
# Sentence embeddings index
self.embeddings = ANNFactory.create(self.config)
- self.embeddings.load("%s/embeddings" % path)
+ self.embeddings.load(f"{path}/embeddings")
# Dimensionality reduction
if self.config.get("pca"):
- with open("%s/lsa" % path, "rb") as handle:
+ with open(f"{path}/lsa", "rb") as handle:
self.reducer = Reducer()
self.reducer.load(path)
@@ -339,7 +339,7 @@ class Embeddings:
true if index exists, false otherwise
"""
- return os.path.exists("%s/config" % path) and os.path.exists("%s/embeddings" % path)
+ return os.path.exists(f"{path}/config") and os.path.exists(f"{path}/embeddings")
def save(self, path):
"""
@@ -360,11 +360,11 @@ class Embeddings:
self.config["path"] = os.path.basename(self.config["path"])
# Write index configuration
- with open("%s/config" % path, "wb") as handle:
+ with open(f"{path}/config", "wb") as handle:
pickle.dump(self.config, handle, protocol=pickle.HIGHEST_PROTOCOL)
# Write sentence embeddings index
- self.embeddings.save("%s/embeddings" % path)
+ self.embeddings.save(f"{path}/embeddings")
# Save dimensionality reduction
if self.reducer:
diff --git a/src/python/txtai/embeddings/documents.py b/src/python/txtai/embeddings/documents.py
index c74efdde..4a9e32b5 100644
--- a/src/python/txtai/embeddings/documents.py
+++ b/src/python/txtai/embeddings/documents.py
@@ -57,6 +57,7 @@ class Documents:
"""
# Create documents file if not already open
+ # pylint: disable=R1732
if not self.documents:
self.documents = tempfile.NamedTemporaryFile(mode="wb", suffix=".docs", delete=False)
diff --git a/src/python/txtai/embeddings/reducer.py b/src/python/txtai/embeddings/reducer.py
index 79dae5da..8b5fa000 100644
--- a/src/python/txtai/embeddings/reducer.py
+++ b/src/python/txtai/embeddings/reducer.py
@@ -83,7 +83,7 @@ class Reducer:
"""
# Dimensionality reduction
- with open("%s/lsa" % path, "rb") as handle:
+ with open(f"{path}/lsa", "rb") as handle:
self.model = pickle.load(handle)
def save(self, path):
@@ -94,5 +94,5 @@ class Reducer:
path: directory path to save model
"""
- with open("%s/lsa" % path, "wb") as handle:
+ with open(f"{path}/lsa", "wb") as handle:
pickle.dump(self.model, handle, protocol=pickle.HIGHEST_PROTOCOL)
diff --git a/src/python/txtai/models/models.py b/src/python/txtai/models/models.py
index c64dc2d2..457e3d37 100644
--- a/src/python/txtai/models/models.py
+++ b/src/python/txtai/models/models.py
@@ -86,7 +86,7 @@ class Models:
device reference
"""
- return "cpu" if deviceid < 0 else "cuda:{}".format(deviceid)
+ return "cpu" if deviceid < 0 else f"cuda:{deviceid}"
@staticmethod
def load(path, config=None, task="default"):
diff --git a/src/python/txtai/models/onnx.py b/src/python/txtai/models/onnx.py
index ef2d1b94..fdd6c58f 100644
--- a/src/python/txtai/models/onnx.py
+++ b/src/python/txtai/models/onnx.py
@@ -66,7 +66,7 @@ class OnnxModel(PreTrainedModel):
# pylint: disable=E1101
# Detect if logits is an output and return classifier output in that case
- if any([x.name for x in self.model.get_outputs() if x.name == "logits"]):
+ if any(x.name for x in self.model.get_outputs() if x.name == "logits"):
return SequenceClassifierOutput(logits=torch.from_numpy(np.array(results[0])))
return torch.from_numpy(np.array(results))
diff --git a/src/python/txtai/pipeline/data/textractor.py b/src/python/txtai/pipeline/data/textractor.py
index c666cb06..d6f80c0f 100644
--- a/src/python/txtai/pipeline/data/textractor.py
+++ b/src/python/txtai/pipeline/data/textractor.py
@@ -46,7 +46,7 @@ class Textractor(Segmentation):
return parsed["content"]
# Fallback to Beautiful Soup
- text = "file://%s" % text if os.path.exists(text) else text
+ text = f"file://{text}" if os.path.exists(text) else text
with contextlib.closing(urlopen(text)) as connection:
text = connection.read()
@@ -68,7 +68,7 @@ class Textractor(Segmentation):
if not path:
path = os.getenv("TIKA_JAVA", "java")
- # pylint: disable=W0702
+ # pylint: disable=R1732,W0702,W1514
# Check if java binary is available on path
try:
_ = Popen(path, stdout=open(os.devnull, "w"), stderr=open(os.devnull, "w"))
diff --git a/src/python/txtai/pipeline/text/extractor.py b/src/python/txtai/pipeline/text/extractor.py
index a2d12f18..6e542818 100644
--- a/src/python/txtai/pipeline/text/extractor.py
+++ b/src/python/txtai/pipeline/text/extractor.py
@@ -133,8 +133,8 @@ class Extractor(Pipeline):
# - all prohibited tokens are not present or there are not prohibited tokens
# - score is above minimum score required
# - number of tokens is above minimum number of tokens required
- if (not must or all([token.lower() in text.lower() for token in must])) and (
- not mnot or all([token.lower() not in text.lower() for token in mnot])
+ if (not must or all(token.lower() in text.lower() for token in must)) and (
+ not mnot or all(token.lower() not in text.lower() for token in mnot)
):
if score >= self.minscore and len(tokenlist[x]) >= self.mintokens:
matches.append(segments[x] + (score,))
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.py
index 78868a29..09333ac9 100644
--- a/src/python/txtai/scoring/base.py
+++ b/src/python/txtai/scoring/base.py
@@ -130,7 +130,7 @@ class Scoring:
path: directory path to load model
"""
- with open("%s/scoring" % path, "rb") as handle:
+ with open(f"{path}/scoring", "rb") as handle:
self.__dict__.update(pickle.load(handle))
def save(self, path):
@@ -141,7 +141,7 @@ class Scoring:
path: directory path to save model
"""
- with open("%s/scoring" % path, "wb") as handle:
+ with open(f"{path}/scoring", "wb") as handle:
pickle.dump(self.__dict__, handle, protocol=pickle.HIGHEST_PROTOCOL)
def computeIDF(self, freq):
diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/words.py
index 683464d4..f93227ca 100644
--- a/src/python/txtai/vectors/words.py
+++ b/src/python/txtai/vectors/words.py
@@ -54,8 +54,6 @@ def transform(document):
(id, embedding)
"""
- global VECTORS
-
return (document[0], VECTORS.transform(document))
@@ -191,12 +189,12 @@ class WordVectors(Vectors):
model = fasttext.train_unsupervised(data, dim=size, minCount=mincount)
# Output file path
- print("Building %d dimension model" % size)
+ print(f"Building {size} dimension model")
# Output vectors in vec/txt format
- with open(path + ".txt", "w") as output:
+ with open(path + ".txt", "w", encoding="utf-8") as output:
words = model.get_words()
- output.write("%d %d\n" % (len(words), model.get_dimension()))
+ output.write(f"{len(words)} {model.get_dimension()}\n")
for word in words:
# Skip end of line token
diff --git a/src/python/txtai/workflow/task/base.py b/src/python/txtai/workflow/task/base.py
index b2c0fa78..36efbb0f 100644
--- a/src/python/txtai/workflow/task/base.py
+++ b/src/python/txtai/workflow/task/base.py
@@ -335,12 +335,12 @@ class Task:
"""
# If all outputs are numpy arrays, use native method
- if all([isinstance(output, np.ndarray) for output in outputs]):
+ if all(isinstance(output, np.ndarray) for output in outputs):
return np.concatenate(np.stack(outputs, axis=1))
# If all outputs are torch tensors, use native method
# pylint: disable=E1101
- if all([torch.is_tensor(output) for output in outputs]):
+ if all(torch.is_tensor(output) for output in outputs):
return torch.cat(tuple(torch.stack(outputs, axis=1)))
# Wrap as one to many transforms
@@ -366,12 +366,12 @@ class Task:
"""
# If all outputs are numpy arrays, use native method
- if all([isinstance(output, np.ndarray) for output in outputs]):
+ if all(isinstance(output, np.ndarray) for output in outputs):
return np.stack(outputs, axis=1)
# If all outputs are torch tensors, use native method
# pylint: disable=E1101
- if all([torch.is_tensor(output) for output in outputs]):
+ if all(torch.is_tensor(output) for output in outputs):
return torch.stack(outputs, axis=1)
return list(zip(*outputs))
diff --git a/src/python/txtai/workflow/task/retrieve.py b/src/python/txtai/workflow/task/retrieve.py
index c3e144e7..41c77b9b 100644
--- a/src/python/txtai/workflow/task/retrieve.py
+++ b/src/python/txtai/workflow/task/retrieve.py
@@ -21,6 +21,7 @@ class RetrieveTask(UrlTask):
# Create default temporary directory if not specified
if not directory:
# Save tempdir to prevent content from being deleted until this task is out of scope
+ # pylint: disable=R1732
self.tempdir = tempfile.TemporaryDirectory()
directory = self.tempdir.name
diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.py
index e416cb0d..1151c247 100644
--- a/test/python/testapi/testcluster.py
+++ b/test/python/testapi/testcluster.py
@@ -90,7 +90,7 @@ class TestCluster(unittest.TestCase):
config = os.path.join(tempfile.gettempdir(), "testapi.yml")
- with open(config, "w") as output:
+ with open(config, "w", encoding="utf-8") as output:
output.write(CLUSTER)
client = TestClient(app)
diff --git a/test/python/testapi/testembeddings.py b/test/python/testapi/testembeddings.py
index a56ad2d4..038dd717 100644
--- a/test/python/testapi/testembeddings.py
+++ b/test/python/testapi/testembeddings.py
@@ -57,7 +57,7 @@ class TestEmbeddings(unittest.TestCase):
config = os.path.join(tempfile.gettempdir(), "testapi.yml")
index = os.path.join(tempfile.gettempdir(), "testapi")
- with open(config, "w") as output:
+ with open(config, "w", encoding="utf-8") as output:
output.write((INDEX if full else READONLY) % (index))
client = TestClient(app)
diff --git a/test/python/testapi/testextension.py b/test/python/testapi/testextension.py
index 4d3d5958..39bb4922 100644
--- a/test/python/testapi/testextension.py
+++ b/test/python/testapi/testextension.py
@@ -82,7 +82,7 @@ class TestExtension(unittest.TestCase):
config = os.path.join(tempfile.gettempdir(), "testapi.yml")
- with open(config, "w") as output:
+ with open(config, "w", encoding="utf-8") as output:
output.write(PIPELINES)
client = TestClient(application.app)
diff --git a/test/python/testapi/testpipelines.py b/test/python/testapi/testpipelines.py
index 6f65ef31..a6b3bb28 100644
--- a/test/python/testapi/testpipelines.py
+++ b/test/python/testapi/testpipelines.py
@@ -61,7 +61,7 @@ class TestPipelines(unittest.TestCase):
config = os.path.join(tempfile.gettempdir(), "testapi.yml")
- with open(config, "w") as output:
+ with open(config, "w", encoding="utf-8") as output:
output.write(PIPELINES)
client = TestClient(app)
@@ -172,7 +172,7 @@ class TestPipelines(unittest.TestCase):
Test summary via API
"""
- summary = self.client.get("summary?text=%s&minlength=15&maxlength=15" % urllib.parse.quote(self.text)).json()
+ summary = self.client.get(f"summary?text={urllib.parse.quote(self.text)}&minlength=15&maxlength=15").json()
self.assertEqual(summary, "txtai is an AI-powered search engine that")
def testSummaryBatch(self):
@@ -188,7 +188,7 @@ class TestPipelines(unittest.TestCase):
Test tabular via API
"""
- results = self.client.get("tabular?file=%s" % Utils.PATH + "/tabular.csv").json()
+ results = self.client.get(f"tabular?file={Utils.PATH}" + "/tabular.csv").json()
# Check length of results is as expected
self.assertEqual(len(results), 6)
@@ -208,7 +208,7 @@ class TestPipelines(unittest.TestCase):
Test textractor via API
"""
- text = self.client.get("textract?file=%s" % Utils.PATH + "/article.pdf").json()
+ text = self.client.get(f"textract?file={Utils.PATH}" + "/article.pdf").json()
# Check length of text is as expected
self.assertEqual(len(text), 2301)
@@ -228,7 +228,7 @@ class TestPipelines(unittest.TestCase):
Test transcribe via API
"""
- text = self.client.get("transcribe?file=%s" % Utils.PATH + "/Make_huge_profits.wav").json()
+ text = self.client.get(f"transcribe?file={Utils.PATH}" + "/Make_huge_profits.wav").json()
# Check length of text is as expected
self.assertEqual(text, "Make huge profits without working make up to one hundred thousand dollars a day")
@@ -248,7 +248,7 @@ class TestPipelines(unittest.TestCase):
Test translate via API
"""
- translation = self.client.get("translate?text=%s&target=es" % urllib.parse.quote("This is a test translation into Spanish")).json()
+ translation = self.client.get(f"translate?text={urllib.parse.quote('This is a test translation into Spanish')}&target=es").json()
self.assertEqual(translation, "Esta es una traducción de prueba al español")
def testTranslateBatch(self):
diff --git a/test/python/testapi/testworkflow.py b/test/python/testapi/testworkflow.py
index b81c307c..897305ba 100644
--- a/test/python/testapi/testworkflow.py
+++ b/test/python/testapi/testworkflow.py
@@ -138,7 +138,7 @@ class TestWorkflow(unittest.TestCase):
config = os.path.join(tempfile.gettempdir(), "testapi.yml")
- with open(config, "w") as output:
+ with open(config, "w", encoding="utf-8") as output:
output.write(WORKFLOWS)
client = TestClient(app)