Develop Reference

Linearize optimization of non-overlapping items along a sequence

This is a follow-up to my previous question here. I have a optimization model that tries to find the highest coverage of a set of probe to a sequence. I approached it by creating an overlap matrix as shown below.
import pyomo
import pyomo.environ as pe
import pyomo.opt as po
import numpy as np
import matplotlib.pyplot as plt
# Initialise all sequences and probes
sequence = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
probes = ["a", "b", "c", "d", "e", "f", "g", "h"]
probe_starts = {"a": 0, "b": 1, "c": 4, "d": 5, "e": 6, "f": 8, "g": 13, "h": 12}
probe_ends = {"a": 2, "b": 2, "c": 6, "d": 6, "e": 8, "f": 11, "g": 15, "h": 14}
probe_lengths = {
p: e - s + 1 for (p, s), e in zip(probe_starts.items(), probe_ends.values())
}
# Create a matrix of probes against probes to check for overlap
def is_overlapping(x, y):
x_start, x_end = x
y_start, y_end = y
return (
(x_start >= y_start and x_start <= y_end)
or (x_end >= y_start and x_end <= y_end)
or (y_start >= x_start and y_start <= x_end)
or (y_end >= x_start and y_end <= x_end)
)
overlap = {}
matrix = np.zeros((len(probes), len(probes)))
for row, x in enumerate(zip(probe_starts.values(), probe_ends.values())):
for col, y in enumerate(zip(probe_starts.values(), probe_ends.values())):
matrix[row, col] = is_overlapping(x, y)
overlap[probes[row]] = list(matrix[row].astype(int))
I now build up my model as normal, adding a constraint that if one probe is assigned than any overlapping probes cannot be assigned.
# Model definition
model = pe.ConcreteModel()
model.probes = pe.Set(initialize=probes)
model.lengths = pe.Param(model.probes, initialize=probe_lengths)
model.overlap = pe.Param(model.probes, initialize=overlap, domain=pe.Any)
model.assign = pe.Var(model.probes, domain=pe.Boolean)
# Objective - highest coverage
obj = sum(model.assign[p] * probe_lengths[p] for p in model.probes)
model.objective = pe.Objective(expr=obj, sense=pe.maximize)
# Constraints
model.no_overlaps = pe.ConstraintList()
for query in model.probes:
model.no_overlaps.add(
sum(
[
model.assign[query] * model.assign[p]
for idx, p in enumerate(model.probes)
if model.overlap[query][idx]
]
)
<= 1
)
This works when solving with the quadratic BONMIN solver as shown below. However, when scaling up to a few thousand probes with significantly more overlap then this becomes prohibitively slowly.
solver = po.SolverFactory("BONMIN")
results = solver.solve(model)
visualize = np.zeros((len(probes), len(sequence)))
for idx, (start, end, val) in enumerate(
zip(probe_starts.values(), probe_ends.values(), model.assign.get_values().values())
):
visualize[idx, start : end + 1] = val + 1
plt.imshow(visualize)
plt.yticks(ticks=range(len(probes)), labels=probes)
plt.xticks(range(len(sequence)))
plt.colorbar()
plt.show()
Any suggestions regarding how to convert this into a linear problem would be appreciated. Thanks in advance!

You can attack this as an Integer Program (IP). There are 2 variables you need: one to indicate whether a probe has been "assigned" and another to indicate (or count) if a spot s in the sequence is covered by probe p in order to do the accounting.
It also helps to chop up the sequence into subsets (shown) that are indexed by the probes which could cover them, if assigned.
There is probably a dynamic programming approach to this as well that somebody might chip in. This works...
Code:
# model to make non-contiguous connections across a sequence
# with objective to "cover" as many points in sequence as possible
import pyomo.environ as pe
sequence = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
probes = ["a", "b", "c", "d", "e", "f", "g", "h"]
probe_starts = {"a": 0, "b": 1, "c": 4, "d": 5, "e": 6, "f": 8, "g": 13, "h": 12}
probe_ends = {"a": 2, "b": 2, "c": 6, "d": 6, "e": 8, "f": 11, "g": 15, "h": 14}
# sequence = [0, 1, 2, 3, 4, 5]
# probes = ["a", "b", "c"]
# probe_starts = {"a": 0, "b": 2, "c": 3}
# probe_ends = {"a": 2, "b": 4, "c": 5}
coverages = {p:[t for t in sequence if t>=probe_starts[p] and t<=probe_ends[p]] for p in probes}
# Model definition
model = pe.ConcreteModel()
model.sequence = pe.Set(initialize=sequence)
model.probes = pe.Set(initialize=probes)
# make an indexed set as convenience of probes:coverage ...
model.covers = pe.Set(model.probes, within=model.sequence, initialize=coverages)
model.covers_flat_set = pe.Set(initialize=[(p,s) for p in probes for s in model.covers[p]])
model.assign = pe.Var(model.probes, domain=pe.Binary) # 1 if probe p is used...
model.covered = pe.Var(model.covers_flat_set, domain=pe.Binary) # s is covered by p
# model.pprint()
# Objective
obj = sum(model.covered[p, s] for (p, s) in model.covers_flat_set)
model.objective = pe.Objective(expr=obj, sense=pe.maximize)
# Constraints
# selected probe must cover the associated points between start and end, if assigned
def cover(model, p):
return sum(model.covered[p, s] for s in model.covers[p]) == len(model.covers[p])*model.assign[p]
model.C1 = pe.Constraint(model.probes, rule=cover)
# cannot cover any point by more than 1 probe
def over_cover(model, s):
cov_options = [(p,s) for p in model.probes if (p, s) in model.covers_flat_set]
if not cov_options:
return pe.Constraint.Skip # no possible coverages
return sum(model.covered[p, s] for (p, s) in cov_options) <= 1
model.C2 = pe.Constraint(model.sequence, rule=over_cover)
solver = pe.SolverFactory('glpk')
result = solver.solve(model)
print(result)
#model.display()
# el-cheapo visualization...
for s in model.sequence:
probe = None
print(f'{s:3d}', end='')
for p in model.probes:
if (p, s) in model.covers_flat_set and model.assign[p].value:
probe = p
if probe:
print(f' {probe}')
else:
print()
Yields:
Problem:
- Name: unknown
Lower bound: 13.0
Upper bound: 13.0
Number of objectives: 1
Number of constraints: 24
Number of variables: 32
Number of nonzeros: 55
Sense: maximize
Solver:
- Status: ok
Termination condition: optimal
Statistics:
Branch and bound:
Number of bounded subproblems: 5
Number of created subproblems: 5
Error rc: 0
Time: 0.007474184036254883
Solution:
- number of solutions: 0
number of solutions displayed: 0
0 a
1 a
2 a
3
4 c
5 c
6 c
7
8 f
9 f
10 f
11 f
12 h
13 h
14 h
15
16
[Finished in 609ms]

Python - filtration for 2D list

I've created a program that can define maximum, minimum and average temperature of 12 months, but I want to extend it by finding months with the hottest and coldest average temperature. My current code is:
def main():
months = [ ["January", 6,3],
["February", 7,3],
["March", 10,4],
["April", 13,6],
["May", 17,9],
["June", 20,12],
["July", 22,14],
["August", 21,14],
["September",19,12],
["October", 14,9],
["November", 10,6],
["December", 7,3] ]
for month in months:
month_name = month[0]
temp = month[1:]
minTemp = min(temp)
avgTemp = sum(temp)/len(temp)
maxTemp = max(temp)
print (month_name, minTemp, avgTemp, maxTemp)
main()
I want to add something like "if month_name have the maximum avg temperature, print month_name and its temperatures. Also if month_name have the minimum avg temperature, print month_name and its temperatures

import math
def main():
months = [ ["January", 6,3],
["February", 7,3],
["March", 10,4],
["April", 13,6],
["May", 17,9],
["June", 20,12],
["July", 22,14],
["August", 21,14],
["September",19,12],
["October", 14,9],
["November", 10,6],
["December", 7,3] ]
mn,mx=math.inf, -math.inf
mx_month=["",0]
mn_month=["",0]
for month in months:
month_name = month[0]
temp = month[1:]
minTemp = min(temp)
avgTemp = sum(temp)/len(temp)
maxTemp = max(temp)
if avgTemp<mn:
mn_month[0],mn_month[1]=month_name,avgTemp
mn=avgTemp
if avgTemp>mx:
mx_month[0],mx_month[1]=month_name,avgTemp
mx=avgTemp
print (month_name, minTemp, avgTemp, avgTemp)
print("Min avg temo month and temp: ",mn_month)
print("Max avg temo month and temp: ",mx_month)
main()

Append the avg to each month and find min/max
months = [["January", 6, 3],
["February", 7, 3],
["March", 10, 4],
["April", 13, 6],
["May", 17, 9],
["June", 20, 12],
["July", 22, 14],
["August", 21, 14],
["September", 19, 12],
["October", 14, 9],
["November", 10, 6],
["December", 7, 3]]
for m in months:
m.append((m[1] + m[2]) / 2)
_min_avg = min(months, key = lambda k: k[3])
_max_avg = max(months, key = lambda k: k[3])
print(_min_avg)
print(_max_avg)

How to get postion range of a list when give any number?

I have list [1, 2, 5, 6, 9, 10, 14, 19], how can i get any number range index.
For example:
l = [1, 2, 5, 6, 9, 10, 14, 19]
value = 11
range_index = get_range_index(l)
range_index = (5, 6) # need like this
# give a value = 11, need to get value index like (5, 6), because 10 < value < 14.
# the list size may be very very long,can there have good method?
This i try to get left value and calculate index by returned left value.
It's not very good and not high performance.
def get_left_point(self, data, value):
if len(data) == 1:
return data[0]
mid_index, mid_value = len(data) // 2, data[len(data) // 2]
if value >= float(mid_value):
ret = self.get_left_point(data[mid_index:], value)
else:
ret = self.get_left_point(data[:mid_index], value)
return ret

my_range = (100, 101)
[x for x in l if (my_range[0] <= x) and (x <= my_range[1])]

i need help getting stDev without using importmath, python

### import math
def mean(values):
return sum(values)*1.0/len(values)
def std():
pass
print(std())
def std(values):
length = len(values)
if length < 2:
return("Standard deviation requires at least two data points")
m = mean(values)
total_sum = 0
for i in range(length):
total_sum += (values[i]-m)**2
under_root = total_sum*1.0/length
return math.sqrt(under_root)
vals = [5]
stan_dev = std(vals)
print(stan_dev)
values = [1, 2, 3, 4, 5]
stan_dev = std(values)
print(stan_dev)
__________________________________________________________________________
lst = [3, 19, 21, 1435, 653342]
sum = reduce((lambda x, y: x +y), lst)
print (sum)
# list = [3, 19, 21, 1435, 653342]
i need to be able to get the stDev without using sum or len
i need to 'unpack' the stDev ???

You can do it with two loops (there are shorter ways but this is simple):
arr = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
# Calculate the mean first
N, X = 0, 0
for xi in arr:
N += 1
X += xi
mean = X/N
# Calculate the standard deviation
DSS = 0
for xi in arr:
DSS += (xi - mean)**2
std = (DSS/N)**(1/2)
Outputs 4.5 for mean and 2.872 for std.

Checking for minute value with datetime in python

lets say i have a start_date
start_date = datetime.time(7, 55, 56)
and i created a time list:
timelist = []
for y in range(0, 24, 1):
for x in range(0, 60, 5):
start = datetime.time(y, x)
timelist.append(start)
timelist = [datetime.time(0, 0),
datetime.time(0, 10),
datetime.time(0, 20),
datetime.time(0, 30),
datetime.time(0, 40),
datetime.time(0, 50),
datetime.time(1, 0),
...]
x.hour returns [0, 1, 2, ..., 23]
x.minute returns [0, 10, 20, 30, 40, 50]
I then want to create a dictionary { [time: 00:00:00, value: 0], [time: 00:10:00, value=0], ... }
now time is x. and the value is created by checking when the start time is:
For instance:
start_date = 07:55:56 → at 07:50:00 value = 1
start_date = 08:03:22 → at 08:00:00 value = 1
start_date = 08:18:00 → at 08:10:00 value = 1
else the value is 0
I need help creating the if statement for checking the minute.
So far I have:
for x in timelist:
if x.hour == start_date.hour:
...

a little bit hard to understand what are asking.
Do you want to know which time interval the start_date belong to?
If so:
timelist = []
for y in range(0, 24, 1):
for x in range(0, 60, 10):
# should be ten?
start = datetime.time(y, x)
timelist.append(start)
timedict = {}
for start in timelist:
timedict[start] = 0
# is it the dictionary looking for?
start_date = datetime.time(7, 55, 56)
for i in range(len(timelist)):
if timelist[i] <= start_date:
if i == len(timelist)-1 or start_date < timelist[i+1]
timedict[timelist[i]] += 1 # or = 1, depend on your usage