MLography/spatial_anomaly.py at master · Scientific-Computing-Lab/MLography · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
import warnings
with warnings.catch_warnings():
    warnings.filterwarnings("ignore",category=FutureWarning)
    # import os
    import numpy as np
    import cv2 as cv
    import matplotlib.pyplot as plt
    import scipy.spatial.distance as dist
    from utils import num_threads, impurity_dist
    import ray
    import time


@ray.remote
def weighted_kth_nn_single(imp_boxes, k_list, imp_area, indices, impurities_chunks):
    impurity_neighbors_and_area = {}

    # weighted kth nn calculation
    for k in k_list:
        impurity_neighbors_and_area[k] = np.zeros(len(impurities_chunks))

    for i in range(len(impurities_chunks)):
        impurity = impurities_chunks[i]
        k_nn = [(imp_area[impurity] / imp_area[x]) ** 4 *
                np.maximum(impurity_dist(imp_boxes[impurity], imp_boxes[x]), 0.00001)
                for x in indices if x != impurity]
        # k_nn = [(imp_area[impurity] ** 6 + imp_area[x] ** 6) *
        #         np.maximum(impurity_dist(imp_boxes[impurity], imp_boxes[x]), 0.00001)
        #         for x in indices if x != impurity]
        # k_nn = [
        #         np.maximum(impurity_dist(imp_boxes[impurity], imp_boxes[x]), 0.00001)
        #         for x in indices if x != impurity]
        k_nn.sort()

        for k in k_list:
            # print("i: "+str(i))
            # print("impurity: " + str(impurity))
            impurity_neighbors_and_area[k][i] = imp_area[impurity] * k_nn[k - 1] ** 2
            # impurity_neighbors_and_area[k][i] = k_nn[k - 1] ** 2
    return impurity_neighbors_and_area


def weighted_kth_nn(imp_boxes, img, markers, k_list, imp_area, indices, need_plot=False):
    # data structure that holds for each impurity it's k nearest neighbor
    # it looks like this: first index: the k nearest neighbor (corresponding to k_list), second index is the impurity.
    start = time.time()
    impurity_neighbors_and_area = {}

    for k in k_list:
        impurity_neighbors_and_area[k] = np.zeros(imp_boxes.shape[0])

    # weighted kth nn calculation
    impurities_chunks = np.array_split(indices, num_threads)

    tasks = list()
    for i in range(num_threads):
        tasks.append(weighted_kth_nn_single.remote(imp_boxes, k_list, imp_area, indices, impurities_chunks[i]))
    for i in range(num_threads):
        task_out = ray.get(tasks[i])
        for k in k_list:
            impurity_neighbors_and_area[k][impurities_chunks[i]] = task_out[k][:]
    end = time.time()
    print("time weighted_kth_nn parallel: " + str(end - start))

    for k in k_list:
        data = impurity_neighbors_and_area[k][indices]
        data[data == 0] = 0.00001
        impurity_neighbors_and_area[k][indices] = np.log(data)
        # impurity_neighbors_and_area[k][indices] = np.maximum(np.log(impurity_neighbors_and_area[k][indices]), 0.00001)

        scores = impurity_neighbors_and_area[k][indices]
        scores = (scores - np.min(scores)) / np.ptp(scores)
        scores = np.maximum(scores - 2 * np.std(scores), 0.00001)

        impurity_neighbors_and_area[k][indices] = (scores - np.min(scores)) / np.ptp(scores)

        # uncomment to see histogram (hope for normal distribution)
        # plt.figure(k)
        # plt.hist(impurity_neighbors_and_area[k][indices])

        max_val2 = max(impurity_neighbors_and_area[k])
        impurity_neighbors_and_area[k] = list(map(lambda x: x / max_val2, impurity_neighbors_and_area[k]))

    if need_plot:
        blank_image2 = {}

        for k in k_list:
            blank_image2[k] = np.zeros(img.shape, np.uint8)
            blank_image2[k][:, :] = (255, 255, 255)
        jet = plt.get_cmap('jet')
        for impurity in indices:
            for k in k_list:
                score = impurity_neighbors_and_area[k][impurity]
                color = jet(score)
                blank_image2[k][markers == impurity + 2] = (color[0] * 255, color[1] * 255, color[2] * 255)

        for i in range(len(k_list)):
            plt.figure(i)
            plt.imshow(blank_image2[k_list[i]], cmap='jet')
            plt.colorbar()
            plt.clim(0, 1)
            plt.title("the kthNN is taken from" + r"$imp$" + " , when the distance to each other impurity" + r"$oth$" +
                      "is calculated in the following manner: " + r"$\log ((\frac{S(imp)}{S(oth)})^2 * box-dist(imp, oth))$"
                      + ", with k = {}".format(k_list[i]))

        plt.show()

    return impurity_neighbors_and_area


def weighted_kth_nn_not_parallel(imp_boxes, img, markers, k_list, imp_area, indices, need_plot=False):
    # data structure that holds for each impurity it's k nearest neighbor
    # it looks like this: first index: the k nearest neighbor (corresponding to k_list), second index is the impurity.

    impurity_neighbors_and_area = {}

    # weighted kth nn calculation
    for k in k_list:
        impurity_neighbors_and_area[k] = np.zeros(imp_boxes.shape[0])

    for impurity in indices:
        k_nn = [(imp_area[impurity] / imp_area[x]) ** 4 * impurity_dist(imp_boxes[impurity], imp_boxes[x])
                for x in indices if x != impurity]
        # k_nn = [impurity_dist(imp_boxes[impurity], imp_boxes[x]) for x in indices if x != impurity]
        k_nn.sort()

        for k in k_list:
            impurity_neighbors_and_area[k][impurity] = imp_area[impurity] * k_nn[k - 1] ** 2
            # impurity_neighbors_and_area[k][impurity] = k_nn[k - 1] ** 2
    print("finished calculating ktn_nn")

    for k in k_list:
        impurity_neighbors_and_area[k][indices] = np.maximum(np.log(impurity_neighbors_and_area[k][indices]), 0.00001)

        scores = impurity_neighbors_and_area[k][indices]
        scores = (scores - np.min(scores)) / np.ptp(scores)
        scores = np.maximum(scores - 2 * np.std(scores), 0.00001)

        impurity_neighbors_and_area[k][indices] = (scores - np.min(scores)) / np.ptp(scores)

        if need_plot:
            plt.figure(k)
            plt.hist(impurity_neighbors_and_area[k][indices])

        max_val2 = max(impurity_neighbors_and_area[k])
        impurity_neighbors_and_area[k] = list(map(lambda x: x / max_val2, impurity_neighbors_and_area[k]))

    # fig = plt.figure(1)

    # plt.show()

    if need_plot:
        blank_image2 = {}

        for k in k_list:
            blank_image2[k] = np.zeros(img.shape, np.uint8)
            blank_image2[k][:, :] = (255, 255, 255)
        jet = plt.get_cmap('jet')
        for impurity in indices:
            for k in k_list:
                score = impurity_neighbors_and_area[k][impurity]
                color = jet(score)
                blank_image2[k][markers == impurity + 2] = (color[0] * 255, color[1] * 255, color[2] * 255)

        for i in range(len(k_list)):
            plt.figure(i)
            plt.imshow(blank_image2[k_list[i]], cmap='jet')
            plt.colorbar()
            plt.clim(0, 1)
            plt.title("the kthNN is taken from" + r"$imp$" + " , when the distance to each other impurity" + r"$oth$" +
                      "is calculated in the following manner: " + r"$\log ((\frac{S(imp)}{S(oth)})^2 * box-dist(imp, oth))$"
                      + ", with k = {}".format(k_list[i]))

        plt.show()

    return impurity_neighbors_and_area