Commit bfaebdb1 authored by Diego André Sant'Ana's avatar Diego André Sant'Ana 🤞

option process or threads

parent e26eda40
......@@ -72,6 +72,8 @@
<option value="E501" />
<option value="W29" />
<option value="E501" />
<option value="W29" />
<option value="E501" />
</list>
</option>
</inspection_tool>
......
This diff is collapsed.
......@@ -12,7 +12,7 @@ python-weka-wrapper
cycler==0.10
cython
h5py
scikit-learn
scikit-learn==0.20.1
statistics
pandas_ml
pyxdg
......
......@@ -14,6 +14,7 @@ import os
import gc
import multiprocessing
from multiprocessing import Process, Manager
import threading
from interface.interface import InterfaceException as IException
from util.file_utils import File
......@@ -43,12 +44,20 @@ class FeatureExtractor(object):
self.extractors = extractors
self.tkParent=tkParent
def extract_all(self, dataset, output_file=None, dirs=None, overwrite=True):
self.data = Manager().list() #is a necessary because have a problem with use Process and normaly declaration
def extract_all(self, dataset, output_file=None, dirs=None, overwrite=True, processor_amd=False):
self.processor_amd=processor_amd
self.threads = []
self.labels = Manager().list()
self.types = Manager().list()
if self.processor_amd == True :
self.data = Manager().list() #is a necessary because have a problem with use Process and normaly declaration
self.labels = Manager().list()
self.types = Manager().list()
else:
self.data = [] #is a necessary because have a problem with use Process and normaly declaration
self.labels = []
self.types = []
"""Runs the feature extraction algorithms on all images of dataset.
Parameters
......@@ -98,12 +107,9 @@ class FeatureExtractor(object):
self.job_extractor(dataset, cl, classes)
self.print_console("Wait a moment, the threads are processing "+str(len(self.threads)) +" images, it may be delayed depending on the size or quantity of the images!")
print("Number of Cores: "+str(multiprocessing.cpu_count()))
#Start threads
with tqdm(total=len(self.threads)) as pbar:
for t in self.threads:
t.start()
pbar.update(1)
pbar.close()
......@@ -135,8 +141,11 @@ class FeatureExtractor(object):
for item in items :
if item.startswith('.'):
continue
#th = threading.Thread(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
th = multiprocessing.Process(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
if self.processor_amd==True :
th = multiprocessing.Process(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
else:
th = threading.Thread(target=self.sub_job_extractor,args=(item, dataset, cl, classes))
self.threads.append(th)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment