auto-scaling available machine pool size

Makefile

@@ -63,3 +63,7 @@ visualize-fsm:
 	cd cmd/metal-api/internal/tools/visualize_fsm
 	go run main.go
 	dot -Tsvg fsm.dot > fsm.svg
+
+.PHONY: mocks
+mocks:
+	docker run --user $$(id -u):$$(id -g) --rm -w /work -v ${PWD}:/work vektra/mockery:v2.21.1 --name MachineManager --dir /work/cmd/metal-api/internal/scaler --output /work/cmd/metal-api/internal/scaler --filename pool_scaler_mock_test.go --testonly --inpackage

cmd/metal-api/internal/datastore/event.go

@@ -5,7 +5,10 @@ import (
 	"log/slog"
 
 	"github.com/metal-stack/metal-api/cmd/metal-api/internal/fsm"
+	"github.com/metal-stack/metal-api/cmd/metal-api/internal/fsm/states"
 	"github.com/metal-stack/metal-api/cmd/metal-api/internal/metal"
+	s "github.com/metal-stack/metal-api/cmd/metal-api/internal/scaler"
+	"github.com/metal-stack/metal-lib/bus"
 )
 
 // ListProvisioningEventContainers returns all machine provisioning event containers.
@@ -40,22 +43,54 @@ func (rs *RethinkStore) UpsertProvisioningEventContainer(ec *metal.ProvisioningE
 	return rs.upsertEntity(rs.eventTable(), ec)
 }
 
-func (rs *RethinkStore) ProvisioningEventForMachine(ctx context.Context, log *slog.Logger, event *metal.ProvisioningEvent, machineID string) (*metal.ProvisioningEventContainer, error) {
-	ec, err := rs.FindProvisioningEventContainer(machineID)
+func (rs *RethinkStore) ProvisioningEventForMachine(ctx context.Context, log *slog.Logger, publisher bus.Publisher, event *metal.ProvisioningEvent, machine *metal.Machine) (*metal.ProvisioningEventContainer, error) {
+	ec, err := rs.FindProvisioningEventContainer(machine.ID)
 	if err != nil && !metal.IsNotFound(err) {
 		return nil, err
 	}
 
 	if ec == nil {
 		ec = &metal.ProvisioningEventContainer{
 			Base: metal.Base{
-				ID: machineID,
+				ID: machine.ID,
 			},
 			Liveliness: metal.MachineLivelinessAlive,
 		}
 	}
 
-	newEC, err := fsm.HandleProvisioningEvent(ctx, log, ec, event)
+	var scaler *s.PoolScaler
+	if machine.PartitionID != "" && machine.SizeID != "" {
+		// in the early lifecycle, when the pxe booting event is submitted
+		// a machine does not have a partition or size , so the pool scaler
+		// cannot work at this stage
+
+		p, err := rs.FindPartition(machine.PartitionID)
+		if err != nil {
+			return nil, err
+		}
+
+		scaler = s.NewPoolScaler(&s.PoolScalerConfig{
+			Log: log.With("pool-scaler", machine.PartitionID),
+			Manager: &manager{
+				rs:          rs,
+				publisher:   publisher,
+				partitionid: p.ID,
+				sizeid:      machine.SizeID,
+			},
+			Partition: *p,
+		})
+	}
+
+	config := states.StateConfig{
+		Log:       log.With("fsm", machine.ID),
+		Context:   ctx,
+		Container: ec,
+		Event:     event,
+		Scaler:    scaler,
+		Machine:   machine,
+	}
+
+	newEC, err := fsm.HandleProvisioningEvent(&config)
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +100,10 @@ func (rs *RethinkStore) ProvisioningEventForMachine(ctx context.Context, log *sl
 	}
 
 	newEC.TrimEvents(100)
-
 	err = rs.UpsertProvisioningEventContainer(newEC)
+	if err != nil {
+		return nil, err
+	}
+
 	return newEC, err
 }

cmd/metal-api/internal/datastore/machine.go

@@ -23,6 +23,7 @@ type MachineSearchQuery struct {
 	Tags        []string `json:"tags" optional:"true"`
 
 	// allocation
+	NotAllocated        *bool       `json:"not_allocated" optional:"true"`
 	AllocationName      *string     `json:"allocation_name" optional:"true"`
 	AllocationProject   *string     `json:"allocation_project" optional:"true"`
 	AllocationImageID   *string     `json:"allocation_image_id" optional:"true"`
@@ -54,7 +55,10 @@ type MachineSearchQuery struct {
 	DiskSizes []int64  `json:"disk_sizes" optional:"true"`
 
 	// state
-	StateValue *string `json:"state_value" optional:"true" enum:"|RESERVED|LOCKED"`
+	HibernationEnabled *bool   `json:"hibernation_enabled" optional:"true"`
+	StateValue         *string `json:"state_value" optional:"true" enum:"|RESERVED|LOCKED"`
+	PreAllocated       *bool   `json:"preallocated" optional:"true"`
+	Waiting            *bool   `json:"waiting" optional:"true"`
 
 	// ipmi
 	IpmiAddress    *string `json:"ipmi_address" optional:"true"`
@@ -114,6 +118,16 @@ func (p *MachineSearchQuery) generateTerm(rs *RethinkStore) *r.Term {
 		})
 	}
 
+	if p.NotAllocated != nil {
+		q = q.Filter(func(row r.Term) r.Term {
+			if *p.NotAllocated {
+				return row.Field("allocation").Eq(nil)
+			} else {
+				return row.Field("allocation").Ne(nil)
+			}
+		})
+	}
+
 	if p.AllocationName != nil {
 		q = q.Filter(func(row r.Term) r.Term {
 			return row.Field("allocation").Field("name").Eq(*p.AllocationName)
@@ -288,12 +302,30 @@ func (p *MachineSearchQuery) generateTerm(rs *RethinkStore) *r.Term {
 		})
 	}
 
+	if p.HibernationEnabled != nil {
+		q = q.Filter(func(row r.Term) r.Term {
+			return row.Field("state").Field("hibernation").Field("enabled").Eq(*p.HibernationEnabled)
+		})
+	}
+
 	if p.StateValue != nil {
 		q = q.Filter(func(row r.Term) r.Term {
 			return row.Field("state").Field("value").Eq(*p.StateValue)
 		})
 	}
 
+	if p.PreAllocated != nil {
+		q = q.Filter(func(row r.Term) r.Term {
+			return row.Field("preallocated").Eq(*p.PreAllocated)
+		})
+	}
+
+	if p.Waiting != nil {
+		q = q.Filter(func(row r.Term) r.Term {
+			return row.Field("waiting").Eq(*p.Waiting)
+		})
+	}
+
 	if p.IpmiAddress != nil {
 		q = q.Filter(func(row r.Term) r.Term {
 			return row.Field("ipmi").Field("address").Eq(*p.IpmiAddress)

cmd/metal-api/internal/datastore/machine_integration_test.go

@@ -367,6 +367,54 @@ func TestRethinkStore_SearchMachines(t *testing.T) {
 			},
 			wantErr: nil,
 		},
+		{
+			name: "search by not_allocated true",
+			q: &MachineSearchQuery{
+				NotAllocated: pointer.Pointer(true),
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, Allocation: &metal.MachineAllocation{}},
+				{Base: metal.Base{ID: "2"}, Allocation: &metal.MachineAllocation{}},
+				{Base: metal.Base{ID: "3"}, Allocation: nil},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "3"}, Allocation: nil}),
+			},
+			wantErr: nil,
+		},
+		{
+			name: "search by not_allocated false",
+			q: &MachineSearchQuery{
+				NotAllocated: pointer.Pointer(false),
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, Allocation: &metal.MachineAllocation{}},
+				{Base: metal.Base{ID: "2"}, Allocation: &metal.MachineAllocation{}},
+				{Base: metal.Base{ID: "3"}, Allocation: nil},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "1"}, Allocation: &metal.MachineAllocation{}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "2"}, Allocation: &metal.MachineAllocation{}}),
+			},
+			wantErr: nil,
+		},
+		{
+			name: "search by not_allocated nil",
+			q: &MachineSearchQuery{
+				NotAllocated: nil,
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, Allocation: &metal.MachineAllocation{}},
+				{Base: metal.Base{ID: "2"}, Allocation: &metal.MachineAllocation{}},
+				{Base: metal.Base{ID: "3"}, Allocation: nil},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "1"}, Allocation: &metal.MachineAllocation{}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "2"}, Allocation: &metal.MachineAllocation{}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "3"}, Allocation: nil}),
+			},
+			wantErr: nil,
+		},
 		{
 			name: "search by network ids",
 			q: &MachineSearchQuery{
@@ -644,6 +692,93 @@ func TestRethinkStore_SearchMachines(t *testing.T) {
 			},
 			wantErr: nil,
 		},
+		{
+			name: "search by waiting true",
+			q: &MachineSearchQuery{
+				Waiting: pointer.Pointer(true),
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, Waiting: true},
+				{Base: metal.Base{ID: "2"}, Waiting: true},
+				{Base: metal.Base{ID: "3"}, Waiting: false},
+				{Base: metal.Base{ID: "4"}, Waiting: false},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "1"}, Waiting: true}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "2"}, Waiting: true}),
+			},
+			wantErr: nil,
+		},
+		{
+			name: "search by preallocated true",
+			q: &MachineSearchQuery{
+				PreAllocated: pointer.Pointer(true),
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, PreAllocated: true},
+				{Base: metal.Base{ID: "2"}, PreAllocated: true},
+				{Base: metal.Base{ID: "3"}, PreAllocated: false},
+				{Base: metal.Base{ID: "4"}, PreAllocated: false},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "1"}, PreAllocated: true}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "2"}, PreAllocated: true}),
+			},
+			wantErr: nil,
+		},
+		{
+			name: "search by hibernation_enabled true",
+			q: &MachineSearchQuery{
+				HibernationEnabled: pointer.Pointer(true),
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}},
+				{Base: metal.Base{ID: "2"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}},
+				{Base: metal.Base{ID: "3"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}},
+				{Base: metal.Base{ID: "4"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "1"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "2"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}}),
+			},
+			wantErr: nil,
+		},
+		{
+			name: "search by hibernation_enabled false",
+			q: &MachineSearchQuery{
+				HibernationEnabled: pointer.Pointer(false),
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}},
+				{Base: metal.Base{ID: "2"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}},
+				{Base: metal.Base{ID: "3"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}},
+				{Base: metal.Base{ID: "4"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "3"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "4"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}}),
+			},
+			wantErr: nil,
+		},
+		{
+			name: "search by hibernation_enabled nil",
+			q: &MachineSearchQuery{
+				HibernationEnabled: nil,
+			},
+			mock: []*metal.Machine{
+				{Base: metal.Base{ID: "1"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}},
+				{Base: metal.Base{ID: "2"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}},
+				{Base: metal.Base{ID: "3"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}},
+				{Base: metal.Base{ID: "4"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}},
+			},
+			want: []*metal.Machine{
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "1"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "2"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: true}}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "3"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}}),
+				tt.defaultBody(&metal.Machine{Base: metal.Base{ID: "4"}, State: metal.MachineState{Hibernation: metal.MachineHibernation{Enabled: false}}}),
+			},
+			wantErr: nil,
+		},
 		{
 			name: "search by ipmi address",
 			q: &MachineSearchQuery{