pynovisao.py 41.6 KB
Newer Older
1 2
#
"""
3 4 5 6
    This file must contain the implementation code for all actions of pynovisao.
    
    Name: pynovisao.py
    Author: Alessandro dos Santos Ferreira ( santosferreira.alessandro@gmail.com )
7 8 9
"""

from collections import OrderedDict
10
import numpy as np
11
import os
12
import interface
13 14
import types
import cv2
15
from interface.interface import InterfaceException as IException
16
from PIL import Image
17

18
import segmentation
19
import extraction
20
from extraction import FeatureExtractor
21

22
import classification
23
from classification import Classifier
24

25
import util
26 27
from extraction.extractor_frame_video import ExtractFM

28
from util.config import Config
29
from util.file_utils import File
30
from util.utils import TimeUtils
31 32
from util.utils import MetricUtils
from util.x11_colors import X11Colors
33

Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
34

35
class Act(object):
36
    """Store all actions of Pynovisao."""
37

Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
38
    def __init__(self, tk, args):
39 40 41 42 43 44 45 46 47
        """Constructor.

        Parameters
        ----------
        tk : Interface
            Pointer to interface that handles UI.
        args : Dictionary
            Arguments of program.
        """
48
        self.tk = tk
49
        self.has_trained = False
50 51 52
        
        self.segmenter = [segmentation._segmenter_list[segmenter].meta for segmenter in segmentation._segmenter_list
                            if segmentation._segmenter_list[segmenter].value == True ][0]()
53 54 55
        
        self.extractors = [extraction._extractor_list[extractor].meta for extractor in extraction._extractor_list
                            if extraction._extractor_list[extractor].value == True ]
56 57 58 59 60 61
        
        try:
            self.classifier = [classification._classifier_list[classifier].meta for classifier in classification._classifier_list
                                if classification._classifier_list[classifier].value == True ][0]()
        except:
            self.classifier = None
62

63 64
        self._image = None
        self._const_image = None
65
        self._mask_image = None
66
        self._image_name = None
67
        self._image_path = None
68
                    
69 70
        self._init_dataset(args["dataset"])
        self._init_classes(args["classes"], args["colors"])
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
71

72
        self._dataset_generator = True
73 74
        self._ground_truth = False
        self._gt_segments = None
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
75
        self.weight_path = None
76

77
    
78
    def _init_dataset(self, directory):
79 80 81 82 83 84 85
        """Initialize the directory of image dataset.

        Parameters
        ----------
        directory : string
            Path to directory.
        """
86 87 88 89
        if(directory[-1] == '/'):
            directory = directory[:-1]
            
        self.dataset = directory
90
        File.create_dir(self.dataset)
91
    
92
    def _init_classes(self, classes = None, colors = None):
93 94 95 96 97 98 99 100 101 102
        """Initialize the classes of dataset.

        Parameters
        ----------
        classes : list of string, optional, default = None
            List of classes. If not informed, the metod set as classes all classes in dataset. 
            If there's no classes in dataset, adds two default classes.
        colors : list of string, optional, default = None
            List de colors representing the color of classe, in same order. If not informed, chooses a color at random.
        """
103
        self.classes = []
104 105 106 107 108 109 110 111 112 113 114 115 116 117

        dataset_description_path = File.make_path(self.dataset, '.dataset_description.txt')

        if os.path.exists(dataset_description_path):
            colors = []
            classes = []
            file = open(dataset_description_path, "r") 
            for line in file:
                class_info = line.replace("\n", "").split(",")
                classes.append(class_info[0])
                colors.append(class_info[1])                 
        else:
            classes = sorted(File.list_dirs(self.dataset)) if classes is None else classes.split()
            colors = [] if colors is None else colors.split()
118 119 120 121 122 123 124

        if(len(classes) > 0):
            for i in range(0, len(classes)):
                self.add_class(dialog = False, name=classes[i], color=colors[i] if i < len(colors) else None)
        else:
            self.add_class(dialog = False, color='Green')
            self.add_class(dialog = False, color='Yellow')
125
                
126
        self._current_class = 0
127
        
128

129
    def open_image(self, imagename = None):
130 131 132 133 134 135 136
        """Open a new image.

        Parameters
        ----------
        imagename : string, optional, default = None
            Filepath of image. If not informed open a dialog to choose.
        """
137 138
        
        def onclick(event):
139
            """Binds dataset generator event to click on image."""
140
            if event.xdata != None and event.ydata != None and int(event.ydata) != 0 and self._dataset_generator == True:
141 142
                x = int(event.xdata)
                y = int(event.ydata)
143 144 145 146 147 148 149
                self.tk.write_log("Coordinates: x = %d y = %d", x, y)
                
                segment, size_segment, idx_segment, run_time = self.segmenter.get_segment(x, y)
                
                if size_segment > 0:
                    self.tk.append_log("\nSegment = %d: %0.3f seconds", idx_segment, run_time)
                    
150
                    self._image, run_time = self.segmenter.paint_segment(self._image, self.classes[self._current_class]["color"].value, x, y)
151
                    self.tk.append_log("Painting segment: %0.3f seconds", run_time)
152
                    self.tk.refresh_image(self._image)
153
                    
154 155 156 157
                    if self._ground_truth == True:
                        self._gt_segments[idx_segment] = self.classes[self._current_class]["name"].value

                    elif self._dataset_generator == True:
158
                        filepath = File.save_class_image(segment, self.dataset, self.classes[self._current_class]["name"].value, self._image_name, idx_segment)
159 160
                        if filepath:
                            self.tk.append_log("\nSegment saved in %s", filepath)
161 162 163
        
        if imagename is None:
            imagename = self.tk.utils.ask_image_name()
164 165

        if imagename:
166 167
            self._image = File.open_image(imagename)
            self._image_name = File.get_filename(imagename)
168

169 170 171
            self.tk.write_log("Opening %s...", self._image_name)
            self.tk.add_image(self._image, self._image_name, onclick)
            self._const_image = self._image
172
            
173
            self.segmenter.reset()
174
            self._gt_segments = None
175

176
        
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
177 178 179 180

    def open_weight(self):
        """Open a new weight."""
        self.weight_path = self.tk.utils.ask_weight_name()
181
        self.classifier.weight_path = self.weight_path
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
182
        
183
    def restore_image(self):
184 185
        """Refresh the image and clean the segmentation.
        """
186 187 188 189
        if self._const_image is not None:
            self.tk.write_log("Restoring image...")
            self.tk.refresh_image(self._const_image)
            
190
            self.segmenter.reset()
191
            self._gt_segments = None
192 193
        
    def close_image(self):
194
        """Close the image.
195
        
196 197 198 199 200
        Raises
        ------
        IException 'Image not found'
            If there's no image opened.
        """
201
        if self._const_image is None:
202 203 204 205
            raise IException("Image not found")
        
        if self.tk.close_image():
            self.tk.write_log("Closing image...")
206
            self._const_image = None
207
            self._image = None
208
            self._image_path = None
209 210

    def add_class(self, dialog = True, name = None, color = None):
211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
        """Add a new class.

        Parameters
        ----------
        dialog : boolean, optional, default = True
            If true open a config dialog to add the class.
        name : string, optional, default = None
            Name of class. If not informed set the name 'Class_nn' to class.
        color : string, optional, default = None
            Name of color in X11Color format, representing the class. It will used to paint the segments of class.
            If not informed choose a color at random.
            
        Raises
        ------
        IException 'You have reached the limite of %d classes'
            If you already have created self.tk.MAX_CLASSES classes.
        """
228 229 230
        n_classes = len(self.classes)
        if n_classes >= self.tk.MAX_CLASSES:
            raise IException("You have reached the limite of %d classes" % self.tk.MAX_CLASSES)
231
                
232
        def edit_class(index):
233
            """Calls method that edit the class."""
234
            self.edit_class(index)
235 236
            
        def update_current_class(index):
237
            """Calls method that update the class."""
238
            self.update_current_class(index)
239 240
        
        def process_config():
241
            """Add the class and refresh the panel of classes."""
242
            new_class = self.tk.get_config_and_destroy()
243
            new_class["name"].value = '_'.join(new_class["name"].value.split())
244 245 246

            self.classes.append( new_class )
            self.tk.write_log("New class: %s", new_class["name"].value)
247
            self.tk.refresh_panel_classes(self.classes, self._current_class)
248
            
249 250
        if name is None:
            name = "Class_%02d" % (n_classes+1)
251
        if color is None:
252
            color = util.X11Colors.random_color()
253 254
            
        class_config = OrderedDict()
255
        class_config["name"] = Config(label="Name", value=name, c_type=str)
256
        class_config["color"] = Config(label="Color (X11 Colors)", value=color, c_type='color')
257 258
        class_config["callback"] = Config(label=None, value=update_current_class, c_type=None, hidden=True)
        class_config["callback_color"] = Config(label=None, value=edit_class, c_type=None, hidden=True)
259 260 261 262 263 264 265
        class_config["args"] = Config(label=None, value=n_classes, c_type=int, hidden=True)
        
        if dialog == False:
            self.classes.append( class_config )
            return 

        title = "Add a new classe"
266 267 268
        self.tk.dialogue_config(title, class_config, process_config)        
      

269
    def edit_class(self, index):
270 271 272 273 274 275 276
        """Edit a class.

        Parameters
        ----------
        index : integer.
            Index of class in list self.classes.
        """
277
        def process_update(index):
278
            """Update the class."""
279
            updated_class = self.tk.get_config_and_destroy()
280
            updated_class["name"].value = '_'.join(updated_class["name"].value.split())
281 282 283
            
            self.classes[index] = updated_class
            self.tk.write_log("Class updated: %s", updated_class["name"].value)
284
            self.tk.refresh_panel_classes(self.classes, self._current_class)
285 286 287 288 289 290
        
        current_config = self.classes[index]
            
        title = "Edit class %s" % current_config["name"].value
        self.tk.dialogue_config(title, current_config, lambda *_ : process_update(index))
            
291
    def update_current_class(self, index):
292 293
        """Update the current class.
        """
294
        self._current_class = index
295 296
        
    def get_class_by_name(self, name):
297 298 299 300
        """Return the index for class.
        
        Parameters
        ----------
301
        name : string
302 303 304 305
            Name of class.
            
        Returns
        -------
306
        index : integer
307 308 309 310 311 312 313
            Index of class in list self.classes.

        Raises
        ------
        Exception 'Class not found'
            If name not found in self.classes.
        """
314 315 316 317 318 319
        name = name.strip()
        
        for cl in self.classes:
            if cl["name"].value == name:
                return cl
        raise Exception("Class not found")
320

321
        
322
    def set_dataset_path(self):
323 324
        """Open a dialog to choose the path to directory of image dataset.
        """
325 326
        directory = self.tk.utils.ask_directory(default_dir = self.dataset)
        if directory:
327
            self._init_dataset(directory)
328 329
            self.tk.write_log("Image dataset defined: %s", self.dataset)
            
330
            self._init_classes()
331
            self.tk.refresh_panel_classes(self.classes)
332
            
333
            if self.classifier: self.classifier.reset()
334
        self.has_trained=False
335
            
336
    def toggle_dataset_generator(self):
337 338
        """Enable/disable the dataset generator on click in image.
        """
339
        self._dataset_generator = not self._dataset_generator
340

341 342
            
    def select_segmenter(self):
343 344
        """Open a dialog to choose the segmenter.
        """
345 346
        title = "Choosing a segmenter"
        self.tk.write_log(title)
347

348
        current_config = segmentation.get_segmenter_config()
349
        
350
        def process_config():
351
            """Update the current segmenter."""
352
            new_config = self.tk.get_config_and_destroy()
353

354 355 356
            self.segmenter = [new_config[segmenter].meta for segmenter in new_config
                                if new_config[segmenter].value == True ][0]()

357
            self.tk.append_log("\nSegmenter: %s\n%s", str(self.segmenter.get_name()), str(self.segmenter.get_summary_config()))
358 359 360 361 362
            segmentation.set_segmenter_config(new_config)

        self.tk.dialogue_choose_one(title, current_config, process_config)

    def config_segmenter(self):
363 364
        """Open a dialog to configure the current segmenter.
        """
365 366 367 368 369 370
        title = "Configuring %s" % self.segmenter.get_name()
        self.tk.write_log(title)

        current_config = self.segmenter.get_config()
        
        def process_config():
371
            """Update the configs of current segmenter."""
372 373 374
            new_config = self.tk.get_config_and_destroy()

            self.segmenter.set_config(new_config)
375
            self.tk.append_log("\nConfig updated:\n%s", str(self.segmenter.get_summary_config()))
376
            self.segmenter.reset()
377 378

        self.tk.dialogue_config(title, current_config, process_config)
379
        
380
    def run_segmenter(self, refresh_image=True):
381
        """Do the segmentation of image, using the current segmenter.
382
        
383 384 385 386 387
        Raises
        ------
        IException 'Image not found'
            If there's no image opened.
        """
388 389 390
        if self._const_image is None:
            raise IException("Image not found")
        
391
        self.tk.write_log("Running %s...", self.segmenter.get_name())
392 393 394 395 396

        self.tk.append_log("\nConfig: %s", str(self.segmenter.get_summary_config()))
        self._image, run_time = self.segmenter.run(self._const_image)
        self.tk.append_log("Time elapsed: %0.3f seconds", run_time)
        
397 398
        self._gt_segments = [None]*(max(self.segmenter.get_list_segments())+1)
        
399 400
        if refresh_image:
            self.tk.refresh_image(self._image)
401 402


403
    def select_extractors(self):
404
        """Open a dialog to select the collection of extractors.
405
        
406 407 408 409 410
        Raises
        ------
        IException 'Please select at least one extractor'
            If no extractor was selected.
        """
411 412 413 414 415 416
        title = "Selecting extractors"
        self.tk.write_log(title)

        current_config = extraction.get_extractor_config()
        
        def process_config():
417
            """Update the collection of extractors."""
418 419 420 421
            new_config = self.tk.get_config_and_destroy()

            self.extractors = [new_config[extractor].meta for extractor in new_config
                                if new_config[extractor].value == True ]
422
            
423 424
            if len(self.extractors) == 0:
                raise IException("Please select at least one extractor")
425
            
426 427 428 429 430 431
            self.tk.append_log("\nConfig updated:\n%s", 
                                '\n'.join(["%s: %s" % (new_config[extractor].label, "on" if new_config[extractor].value==True else "off")
                                            for extractor in new_config]))
            extraction.set_extractor_config(new_config)

        self.tk.dialogue_select(title, current_config, process_config)
432 433
        
    def run_extractors(self):
434 435
        """Perform a feature extraction on all images of dataset, using the current collection of extractors.
        """
436
        self.tk.write_log("Running extractors on all images in %s", self.dataset)
437

438 439 440
        fextractor = FeatureExtractor(self.extractors)
        self.tk.append_log("%s", '\n'.join([extraction._extractor_list[extractor].label for extractor in extraction._extractor_list
                                                if extraction._extractor_list[extractor].value == True ]))
441
        
442
        output_file, run_time = fextractor.extract_all(self.dataset, "training")
443 444
        self.tk.append_log("\nOutput file saved in %s", output_file)
        self.tk.append_log("Time elapsed: %0.3f seconds", run_time)
445 446
        
        if self.classifier: self.classifier.reset()
447

448 449 450 451 452
    def run_extract_frame(self):
        self.tk.write_log("Running extract frames from videos")
        extract_frame=ExtractFM()
        extract_frame.run(self.tk)

453
    def select_classifier(self):
454 455 456 457 458 459 460
        """Open a dialog to select the classifier.
        
        Raises
        ------
        IException 'You must install python-weka-wrapper'
            The user must install the required dependencies to classifiers.
        """
461
        if self.classifier is None:
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
462
            raise IException("Classifier not found!")
463 464 465 466 467
        
        title = "Choosing a classifier"
        self.tk.write_log(title)

        current_config = classification.get_classifier_config()
468

469 470
        
        def process_config():
471
            """Update the current classifier."""
472
            new_config = self.tk.get_config_and_destroy()
Geazy Menezes's avatar
Geazy Menezes committed
473
            
474 475 476 477 478 479 480 481 482
            self.classifier = [new_config[classifier].meta for classifier in new_config
                                if new_config[classifier].value == True ][0]()

            self.tk.append_log("\nClassifier: %s\n%s", str(self.classifier.get_name()), str(self.classifier.get_summary_config()))
            classification.set_classifier_config(new_config)

        self.tk.dialogue_choose_one(title, current_config, process_config)
        
    def configure_classifier(self):
483 484 485 486 487 488 489
        """Set the configuration of current classifier.
        
        Raises
        ------
        IException 'You must install python-weka-wrapper'
            The user must install the required dependencies to classifiers.
        """
490
        if self.classifier is None:
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
491
            raise IException("Classifier not found!")
492 493 494 495 496 497 498 499 500 501 502
        
        title = "Configuring %s" % self.classifier.get_name()
        self.tk.write_log(title)

        current_config = self.classifier.get_config()
        
        def process_config():
            new_config = self.tk.get_config_and_destroy()

            self.classifier.set_config(new_config)
            self.tk.append_log("\nConfig updated:\n%s", str(self.classifier.get_summary_config()))
503 504
            
            if self.classifier: self.classifier.reset()
505 506 507 508 509

        self.tk.dialogue_config(title, current_config, process_config)
    
    
    def run_classifier(self):
510 511 512 513 514 515 516 517 518 519
        """Run the classifier on the current image.
        As result, paint the image with color corresponding to predicted class of all segment.
        
        Raises
        ------
        IException 'You must install python-weka-wrapper'
            The user must install the required dependencies to classifiers.
        IException 'Image not found'
            If there's no image opened.
        """
520
        if self.classifier is None:
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
521
            raise IException("Classifier not found!")
522 523 524 525 526 527 528
        
        if self._const_image is None:
            raise IException("Image not found")
        
        self.tk.write_log("Running %s...", self.classifier.get_name())
        self.tk.append_log("\n%s", str(self.classifier.get_summary_config()))
        
529 530
        #self.classifier.set
        
531 532
        start_time = TimeUtils.get_time()

533
        # Perform a segmentation, if needed.
534 535 536 537 538 539 540
        list_segments = self.segmenter.get_list_segments()
        if len(list_segments) == 0:
            self.tk.append_log("Running %s... (%0.3f seconds)", self.segmenter.get_name(), (TimeUtils.get_time() - start_time))
            
            self._image, _ = self.segmenter.run(self._const_image)
            self.tk.refresh_image(self._image)        
            list_segments = self.segmenter.get_list_segments()
541
            self._gt_segments = [None]*(max(list_segments)+1)
542 543
        
        #  New and optimized classification
544
        tmp = ".tmp"
545
        File.remove_dir(File.make_path(self.dataset, tmp))
546

547 548 549 550 551
        self.tk.append_log("Generating test images... (%0.3f seconds)", (TimeUtils.get_time() - start_time))
        
        len_segments = {}
        for idx_segment in list_segments:
            segment, size_segment, idx_segment = self.segmenter.get_segment(self, idx_segment=idx_segment)[:-1]
552
            
553
            filepath = File.save_class_image(segment, self.dataset, tmp, self._image_name, idx_segment)
554
            len_segments[idx_segment] = size_segment
555
            
556
        # Perform the feature extraction of all segments in image ( not applied to ConvNets ).
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
557
        if self.classifier.must_extract_features():
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
558 559
            self.tk.append_log("Running extractors on test images... (%0.3f seconds)", (TimeUtils.get_time() - start_time))    
            fextractor = FeatureExtractor(self.extractors)        
560 561 562
            output_file, _ = fextractor.extract_all(self.dataset, "test", dirs=[tmp])
                
        self.tk.append_log("Running classifier on test data... (%0.3f seconds)", (TimeUtils.get_time() - start_time))
563 564

        # Get the label corresponding to predict class for each segment of image.
565 566
        labels = self.classifier.classify(self.dataset, test_dir=tmp, test_data="test.arff", image=self._const_image)
        File.remove_dir(File.make_path(self.dataset, tmp))
567

568 569 570 571 572 573 574
        # Result is the class for each superpixel
        if type(labels) is types.ListType:
            self.tk.append_log("Painting segments... (%0.3f seconds)", (TimeUtils.get_time() - start_time))
            
            # If ground truth mode, show alternative results
            if self._ground_truth == True:
                return self._show_ground_truth(list_segments, len_segments, labels, start_time)
575

576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
            # Create a popup with results of classification.
            popup_info = "%s\n" % str(self.classifier.get_summary_config())
            
            len_total = sum([len_segments[idx] for idx in len_segments])
            popup_info += "%-16s%-16s%0.2f%%\n" % ("Total", str(len_total), (len_total*100.0)/len_total)
            
            # Paint the image.
            self._mask_image = np.zeros(self._const_image.shape[:-1], dtype="uint8")
            height, width, channels = self._image.shape
            self.class_color = np.zeros((height,width,3), np.uint8)
            for (c, cl) in enumerate(self.classes):
                idx_segment = [ list_segments[idx] for idx in range(0, len(labels)) if cl["name"].value == labels[idx]]
                if len(idx_segment) > 0:
                    self._image, _ = self.segmenter.paint_segment(self._image, cl["color"].value, idx_segment=idx_segment, border=False)
                    for idx in idx_segment:
                        self._mask_image[self.segmenter._segments == idx] = c
                        self.class_color[self.segmenter._segments == idx] = X11Colors.get_color(cl["color"].value)
                  
                len_classes = sum([len_segments[idx] for idx in idx_segment])
                popup_info += "%-16s%-16s%0.2f%%\n" % (cl["name"].value, str(len_classes), (len_classes*100.0)/len_total)


            self.tk.refresh_image(self._image)
            self.tk.popup(popup_info)
        else:
            # Result is an image
            self._mask_image = labels
            height, width, channels = self._image.shape
            self.class_color = np.zeros((height,width,3), np.uint8)

            for (c, cl) in enumerate(self.classes):
                self.class_color[labels == c] = X11Colors.get_color(cl["color"].value)

            self._image = cv2.addWeighted(self._const_image, 0.7, self.class_color, 0.3, 0)
            self.tk.refresh_image(self._image)
611 612

        
613 614 615 616
        end_time = TimeUtils.get_time()
            
        self.tk.append_log("\nClassification finished")
        self.tk.append_log("Time elapsed: %0.3f seconds", (end_time - start_time))
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
617 618 619

    def run_training(self):
        start_time = TimeUtils.get_time()
620
        
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
621 622 623 624 625 626 627 628 629 630 631 632
        if self._const_image is None:
            raise IException("Image not found")
        
        if self.classifier.must_train():
            
            if self.classifier.must_extract_features():
                self.tk.append_log("Creating training data... (%0.3f seconds)", (TimeUtils.get_time() - start_time))
                fextractor = FeatureExtractor(self.extractors)
                output_file, run_time = fextractor.extract_all(self.dataset, "training", overwrite = False)
        
            self.tk.append_log("Training classifier...")
            
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
633
            self.classifier.train(self.dataset, "training")
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
634 635 636 637

            self.tk.append_log("DONE (%0.3f seconds)",  (TimeUtils.get_time() - start_time))
        
        self._image = self._const_image
638
        self.has_trained=True
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
639

640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
    
    def _show_ground_truth(self, list_segments, len_segments, labels, start_time):
        """Paint only wrong classified segments and show ground truth confusion matrix.
        
        Parameters
        ----------
        list_segments : list of integer
            List of index segments.
        len_segments : list of integer
            List of segments sizes.
        labels : list of string
            List of predicted class name for each segment.
        start_time : floating point
            Start time of classification.
        """
        classes = list(set(labels))
        classes.sort()
        
        n_segments = len(labels)
        spx_matrix = np.zeros((len(classes), len(classes)), np.int) 
        px_matrix = np.zeros((len(classes), len(classes)), np.int) 

        # Create the confusion matrix and paint wrong classified segments individually.
        for idx_segment in list_segments:
            if self._gt_segments[idx_segment] is not None:
                gt_class = classes.index(self._gt_segments[idx_segment])
                predicted_class = classes.index(labels[idx_segment])
                
                spx_matrix[ gt_class ][ predicted_class ] += 1
                px_matrix[ gt_class ][ predicted_class ] += len_segments[idx_segment]
        
                if gt_class != predicted_class:
                    self._image, _ = self.segmenter.paint_segment(self._image, self.get_class_by_name(labels[idx_segment])["color"].value, idx_segment=[idx_segment], border=False)
        
        # Create a popup with results of classification.
        popup_info = "%s\n" % str(self.classifier.get_summary_config())
        popup_info += Classifier.confusion_matrix(classes, spx_matrix, "Superpixels")
        popup_info += Classifier.confusion_matrix(classes, px_matrix, "PixelSum")
        
        self.tk.refresh_image(self._image)
        self.tk.popup(popup_info)
681

682 683 684 685 686
        end_time = TimeUtils.get_time()
            
        self.tk.append_log("\nClassification finished")
        self.tk.append_log("Time elapsed: %0.3f seconds", (end_time - start_time))
        
687

688 689 690 691 692
    def toggle_ground_truth(self):
        """Enable/disable ground truth mode.
        """
        self._ground_truth = not self._ground_truth
        
693
    def cross_validation(self):
694 695 696 697 698 699 700
        """Run a cross validation on all generated segments in image dataset.
        
        Raises
        ------
        IException 'You must install python-weka-wrapper'
            The user must install the required dependencies to classifiers.
        """
701
        if self.classifier is None:
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
702
            raise IException("Classifier not found!")
703 704 705 706 707 708 709 710 711 712 713 714 715 716
        
        if self.classifier.must_train():
            self.tk.write_log("Creating training data...")
            
            fextractor = FeatureExtractor(self.extractors)
            output_file, run_time = fextractor.extract_all(self.dataset, "training", overwrite = False)
            self.classifier.train(self.dataset, "training")
        
        self.tk.write_log("Running Cross Validation on %s...", self.classifier.get_name())
        self.tk.append_log("\n%s", str(self.classifier.get_summary_config()))
        
        popup_info = self.classifier.cross_validate()
        self.tk.append_log("Cross Validation finished")
        self.tk.popup(popup_info)
717 718
        
    def experimenter_all(self):
719 720 721 722 723 724 725
        """Perform a test in all availabel classifiers e show the results.
        
        Raises
        ------
        IException 'You must install python-weka-wrapper'
            The user must install the required dependencies to classifiers.
        """
726
        if self.classifier is None:
Gabriel Kirsten's avatar
 
Gabriel Kirsten committed
727
            raise IException("Classifier not found!")
728 729 730 731 732 733 734 735 736 737 738 739 740 741
        
        if self.tk.ask_ok_cancel("Experimenter All", "This may take several minutes to complete. Are you sure?"):
            if self.classifier.must_train():
                self.tk.write_log("Creating training data...")
                
                fextractor = FeatureExtractor(self.extractors)
                output_file, run_time = fextractor.extract_all(self.dataset, "training", overwrite = False)
                self.classifier.train(self.dataset, "training")
                
            self.tk.write_log("Running Experimenter All on %s...", self.classifier.get_name())
            
            popup_info = self.classifier.experimenter()
            self.tk.append_log("\nExperimenter All finished")
            self.tk.popup(popup_info)
742 743


744 745 746 747
    def about(self):
        self.tk.show_info("Pynovisao\n\nVersion 1.0.0\n\nAuthors:\nAlessandro Ferreira\nHemerson Pistori")
        
            
748
    def func_not_available(self):
749
        """Use this method to bind menu options not available."""
750
        self.tk.write_log("This functionality is not available right now.")
751

752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
    def assign_using_labeled_image(self, imagename = None, refresh_image=True):
        """Open a new image.

        Parameters
        ----------
        imagename : string, optional, default = None
            Filepath of image. If not informed open a dialog to choose.
        """

        if len(self.segmenter.get_list_segments()) == 0:
            self.tk.write_log("Error: Image not segmented")
            return

        if self._image is None:
            self.tk.write_log("Error: Open the image to be targeted")
            return

        if imagename is None:
            imagename = self.tk.utils.ask_image_name()

        if imagename:
            self._image_gt = File.open_image_lut(imagename)
            self._image_gt_name = File.get_filename(imagename)

            self.tk.write_log("Opening %s...", self._image_gt_name)

            qtd_classes = len(self.classes)
            qtd_superpixel = len(self.segmenter.get_list_segments())

        tam_gt = self._image_gt.shape
        tam_im = self._image.shape
        if len(tam_gt) > 2:
            self.tk.write_log("Color image is not supported. You must open a gray-scale image")
            return

        if tam_gt[0] != tam_im[0] or tam_gt[1] != tam_im[1]:
            self.tk.write_log("Images with different sizes")
            return
            
        #hist_classes_superpixels = np.zeros((qtd_superpixel, qtd_classes), np.int)      
    
        #for i in range(0, tam_gt[0]):
        #    for j in range(0, tam_gt[1]):          
        #        class_pixel = self._image_gt[i,j]
        #        if class_pixel > qtd_classes:
        #            self.tk.write_log("There is no class for the pixel [%d,%d] = %d on the image", i, j, class_pixel)
        #        else:
        #            #segment, size_segment, idx_segment, run_time = self.segmenter.get_segment(px = j, py = i)
        #            idx_segment = self.segmenter._segments[i, j]
        #            hist_classes_superpixels[idx_segment, class_pixel] = hist_classes_superpixels[idx_segment, class_pixel] + 1
        #    if i % 10 == 0:
        #        self.tk.write_log("Annotating row %d of %d", i, tam_gt[0])
                
        qtd_bad_superpixels = 0
        
        for idx_segment in range(0, qtd_superpixel):
            hist_classes_superpixels = np.histogram(self._image_gt[self.segmenter._segments == idx_segment], bins=range(0,len(self.classes)+1))[0]

            idx_class = np.argmax(hist_classes_superpixels)
            sum_vector = np.sum(hist_classes_superpixels)
            if refresh_image:
                self._image, run_time = self.segmenter.paint_segment(self._image, self.classes[idx_class]["color"].value, idx_segment = [idx_segment])
            #self.tk.append_log("posicao maior = %x  --  soma vetor %d", x, sum_vector)
            if hist_classes_superpixels[idx_class]/sum_vector < 0.5:
                qtd_bad_superpixels = qtd_bad_superpixels + 1

            if self._ground_truth == True:
                self._gt_segments[idx_segment] = self.classes[self._current_class]["name"].value

            elif self._dataset_generator == True:
                if idx_segment % 10 == 0:
                    self.tk.write_log("Saving %d of %d", (idx_segment+1), qtd_superpixel)

                segment, size_segment, idx_segment, run_time = self.segmenter.get_segment(idx_segment = idx_segment)
                filepath = File.save_class_image(segment, self.dataset, self.classes[idx_class]["name"].value, self._image_name, idx_segment)
                if filepath:
                    self.tk.append_log("\nSegment saved in %s", filepath)

        self.tk.refresh_image(self._image)
        self.tk.write_log("%d bad annotated superpixels of %d superpixel (%0.2f)", qtd_bad_superpixels, qtd_superpixel, (float(qtd_bad_superpixels)/qtd_superpixel)*100)



    def run_segmenter_folder(self, foldername=None):

        if foldername is None:
            foldername = self.tk.utils.ask_directory()

        valid_images_extension = ['.jpg', '.png', '.gif', '.jpeg', '.tif']

        fileimages = [name for name in os.listdir(foldername)
                    if os.path.splitext(name)[-1].lower() in valid_images_extension]

        for (i,file) in enumerate(fileimages):
            path_file = os.path.join(foldername, file)
            self.open_image(path_file)
            self.run_segmenter(refresh_image=False)
            label_image = (os.path.splitext(file)[-2] + '_json')
            self.assign_using_labeled_image(os.path.join(foldername, label_image, 'label.png'), refresh_image=False)
            self.tk.write_log("%d of %d images", i, len(fileimages))

    def run_classifier_folder(self, foldername=None):

        if self.classifier is None:
            raise IException("Classifier not found!")

        if foldername is None:
            foldername = self.tk.utils.ask_directory()

        valid_images_extension = ['.jpg', '.png', '.gif', '.jpeg', '.tif']

        fileimages = [name for name in os.listdir(foldername)
                    if os.path.splitext(name)[-1].lower() in valid_images_extension]

        fileimages.sort()

        all_accuracy = []
        all_IoU = []
        all_frequency_weighted_IU = []

        for file in fileimages:
            path_file = os.path.join(foldername, file)
            self.open_image(path_file)
            self.run_classifier()
            label_image = os.path.join(foldername, (os.path.splitext(file)[-2] + '_json'), 'label.png')
            self._image_gt = File.open_image_lut(label_image)
            self._image_gt_name = File.get_filename(label_image)

            tam_gt = self._image_gt.shape
            tam_im = self._mask_image.shape
            if len(tam_gt) > 2:
                self.tk.write_log("Color image is not supported. You must open a gray-scale image")
                return

            if tam_gt[0] != tam_im[0] or tam_gt[1] != tam_im[1]:
                self.tk.write_log("Images with different sizes")
                return

            
            confusion_matrix = MetricUtils.confusion_matrix(self._mask_image, self._image_gt)
            [mean_accuracy, accuracy] = MetricUtils.mean_accuracy(self._mask_image, self._image_gt)
            [mean_IoU, IoU] = MetricUtils.mean_IU(self._mask_image, self._image_gt)
            frequency_weighted_IU = MetricUtils.frequency_weighted_IU(self._mask_image, self._image_gt)

            print('Matriz de Confusao')
            print(confusion_matrix)

            print('Mean Pixel Accuracy')
            print(mean_accuracy)

            print('Pixel accuracy per class')
            print(accuracy)

            print('Mean Intersction over Union')
            print(mean_IoU)

            print('Intersction over Union per class')
            print(IoU)

            print('Frequency Weighted IU')
            print(frequency_weighted_IU)

            all_accuracy.append(accuracy)
            all_IoU.append(IoU)
            all_frequency_weighted_IU.append(frequency_weighted_IU)

            if not os.path.exists("../models_results/"):
                os.makedirs("../models_results/")
            
            path = File.make_path("../models_results/" + file + ".txt")
            path_img = File.make_path("../models_results/" + file + "_seg1.tif")
            path_img2 = File.make_path("../models_results/" + file + "_seg2.tif")

            img = Image.fromarray(self._image)
            img.save(path_img)
            img = Image.fromarray(self.class_color)
            img.save(path_img2)
            
            f=open(path,'ab')
            np.savetxt(f, ['Matriz de confusao'], fmt='%s')
            np.savetxt(f, confusion_matrix, fmt='%.5f')
            np.savetxt(f, ['\nAcuracia'], fmt='%s')
            np.savetxt(f, accuracy, fmt='%.5f')
            np.savetxt(f, ['\nInterseccao sobre uniao'], fmt='%s')
            np.savetxt(f, IoU, fmt='%.5f')
            np.savetxt(f, ['\nInterseccao sobre uniao com peso'], fmt='%s')
            np.savetxt(f, [frequency_weighted_IU], fmt='%.5f')
            f.close()


        path = File.make_path("../models_results/all_metrics.txt")
        f=open(path,'ab')
        np.savetxt(f, ['All Acuracia'], fmt='%s')
        np.savetxt(f, all_accuracy, fmt='%.5f')
        np.savetxt(f, ['\nAll IoU'], fmt='%s')
        np.savetxt(f, all_IoU, fmt='%.5f')
        np.savetxt(f, ['\nAll Frequency Weighted IU'], fmt='%s')
        np.savetxt(f, all_frequency_weighted_IU, fmt='%.5f')
        f.close()

952 953
    def run_grafic_confusion_matrix(self):
        '''
Geazy Menezes's avatar
Geazy Menezes committed
954
        Generate a a graphical confusion matrix where images are classified and according to classification go to the wrong or right folder.
955
        Just Available to WekaClassifier and CNNKeras.
956
        '''
957 958 959 960 961 962 963
        from classification import WekaClassifiers, CNNKeras
        
        is_weka = isinstance(self.classifier, WekaClassifiers)
        is_keras = isinstance(self.classifier, CNNKeras)
        if not (is_weka or is_keras):
            message='Only available to Weka and CNN Keras classifiers.'
            raise IException(message)
Geazy Menezes's avatar
Geazy Menezes committed
964

965 966 967
        
        if not self.has_trained:
            message='Dataset Must Be Trained.'
968
            raise IException(message)
969
        
970
        from os.path import abspath, isdir
971

972 973 974 975
        folder = self.tk.utils.ask_directory()
        if not folder:
            message = 'No selected directory.'
            raise IException(message)
976
            return
977 978
            
        folder = abspath(folder)
979
        dataset = abspath(self.dataset)
980
        if folder == self.dataset:
981 982 983 984 985
            title = 'Same Dataset'
            message = 'The dataset selected is the same of the trained. Are you sure that is right?'
            option=self.tk.ask_ok_cancel(title, message)
            if not option:
                return
986 987 988 989 990 991 992 993
                
        from os import listdir, mkdir
        listdirs=listdir(folder)
        size_dirs = reduce(lambda a,b: a+b, [0]+[len(listdir(folder+'/'+d)) for d in listdirs if isdir(folder+'/'+d)])
        if not size_dirs:
            message = 'Dataset has no content or the subfolder has no content.'
            raise IException(message)
            
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
        from shutil import rmtree
        from os import symlink
        
        def create_folder_struct(matrix_path, class_names, human, computer):
            try:
                rmtree(matrix_path)
            except Exception as e:
                pass
            mkdir(matrix_path, 0777)
            for class_ in class_names:
                real=matrix_path+human+class_+'/'
                mkdir(real, 0777)
                for _class in class_names:
                    mkdir(real+computer+_class, 0777)

1009 1010 1011 1012 1013

        header_output = 'Starting Graphical Confusion Matrix\n\n