Bandits.ArrayModel.hasCondDistrib

`hasCondDistrib_reward'`🔗

LemmaBandits.ArrayModel.hasCondDistrib_reward'

Details

The conditional distribution of the reward at time n + 1, given the history up to time n and the action at time n + 1, is equal to the kernel ν.

theorem

Bandits.ArrayModel.hasCondDistrib_reward'.{u_1, u_2} {𝓐 : Type u_1}
  {R : Type u_2} {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R}
  [Nonempty 𝓐] [StandardBorelSpace 𝓐] [DecidableEq 𝓐] [Countable 𝓐]
  [StandardBorelSpace R] [Nonempty R] (alg : Learning.Algorithm 𝓐 R)
  (ν : ProbabilityTheory.Kernel 𝓐 R)
  [ProbabilityTheory.IsMarkovKernel ν] (n : ℕ) :
  ProbabilityTheory.HasCondDistrib (reward alg (n + 1))
    (fun ω => (hist alg ω n, action alg (n + 1) ω))
    (ProbabilityTheory.Kernel.prodMkLeft (↥(Finset.Iic n) → 𝓐 × R) ν)
    (arrayMeasure ν)
Bandits.ArrayModel.hasCondDistrib_reward'.{u_1,
    u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐]
  [Countable 𝓐] [StandardBorelSpace R]
  [Nonempty R]
  (alg : Learning.Algorithm 𝓐 R)
  (ν : ProbabilityTheory.Kernel 𝓐 R)
  [ProbabilityTheory.IsMarkovKernel ν]
  (n : ℕ) :
  ProbabilityTheory.HasCondDistrib
    (reward alg (n + 1))
    (fun ω =>
      (hist alg ω n,
        action alg (n + 1) ω))
    (ProbabilityTheory.Kernel.prodMkLeft
      (↥(Finset.Iic n) → 𝓐 × R) ν)
    (arrayMeasure ν)

Code

lemma hasCondDistrib_reward' (alg : Algorithm 𝓐 R) (ν : Kernel 𝓐 R) [IsMarkovKernel ν] (n : ℕ) :
    HasCondDistrib (reward alg (n + 1)) (fun ω ↦ (hist alg ω n, action alg (n + 1) ω))
      (ν.prodMkLeft _) (arrayMeasure ν)

Type uses (7)

Body uses (16)

Used by (1)

hasCondDistrib_reward

Actions: Source · Open Issue

Proof

by
  let R' := reward alg (n + 1)
  let H := (hist alg · n)
  let A := action alg (n + 1)
  let P := fun ω ↦ pullCount (action alg) (action alg (n + 1) ω) (n + 1) ω
  have hP : Measurable P := measurable_pullCount_action_add_one alg n
  change HasCondDistrib R' (fun ω ↦ (H ω, A ω)) (ν.prodMkLeft _) _
  suffices HasCondDistrib R' (fun ω ↦ (A ω, H ω)) (ν.prodMkRight _) (arrayMeasure ν) by
    have h_eq : (fun ω ↦ (H ω, A ω)) = MeasurableEquiv.prodComm ∘ (fun ω ↦ (A ω, H ω)) := rfl
    rw [h_eq]
    exact this.measurableEquiv_comp_right (κ := ν.prodMkRight _) _
  suffices HasCondDistrib R' (fun ω ↦ ((A ω, H ω), P ω))
      ((ν.prodMkRight _).prodMkRight _) (arrayMeasure ν) by
    -- use that `P` is measurable wrt `(A, H)` to drop it from the conditioning
    have hP_meas :
        Measurable[MeasurableSpace.comap (fun ω ↦ (A ω, H ω)) inferInstance] P :=
      measurable_pullCount_action_add_one_hist alg n
    obtain ⟨f, hf_meas, hf_eq⟩ := hP_meas.exists_eq_measurable_comp
    simp only [hf_eq, Function.comp_apply] at this
    rwa [hasCondDistrib_prod_right_iff _ _ hf_meas] at this
  suffices HasCondDistrib R' (fun ω ↦ ((A ω, P ω), H ω))
      ((ν.prodMkRight _).prodMkRight _) (arrayMeasure ν) by
    let e : ((𝓐 × ℕ) × (Iic n → 𝓐 × R)) ≃ᵐ ((𝓐 × (Iic n → 𝓐 × R)) × ℕ) :=
    { toFun := fun x ↦ ((x.1.1, x.2), x.1.2)
      invFun := fun x ↦ ((x.1.1, x.2), x.1.2)
      measurable_toFun := by simp only [Equiv.coe_fn_mk]; fun_prop
      measurable_invFun := by simp only [Equiv.symm_mk, Equiv.coe_fn_mk]; fun_prop }
    exact this.measurableEquiv_comp_right e
  suffices HasCondDistrib R' (fun ω ↦ (A ω, P ω)) (ν.prodMkRight _) (arrayMeasure ν) by
    have h_indep : H ⟂ᵢ[(fun ω ↦ (A ω, P ω)), (by fun_prop); arrayMeasure ν] R' :=
      (condIndepFun_reward_hist alg ν n).symm
    have h_condDistrib := this.condDistrib_eq
    rw [condIndepFun_iff_condDistrib_prod_ae_eq_prodMkRight (by fun_prop) (by fun_prop)
      (by fun_prop)] at h_indep
    refine hasCondDistrib_of_condDistrib_eq (by fun_prop) (by fun_prop) ?_
    refine h_indep.trans ?_
    rw [Filter.EventuallyEq, ae_map_iff] at h_condDistrib ⊢
    · simpa only [Kernel.prodMkRight_apply]
    · fun_prop
    · exact Kernel.measurableSet_eq _ _
    · fun_prop
    · exact Kernel.measurableSet_eq _ _
  exact hasCondDistrib_reward_pullCount_action alg ν n

`Algorithm`🔗

StructureLearning.Algorithm

Details

A stochastic, sequential algorithm.

structure

Learning.Algorithm.{u_4, u_5} (𝓐 : Type u_4) (𝓨 : Type u_5)
  [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] : Type (max u_4 u_5)
Learning.Algorithm.{u_4, u_5}
  (𝓐 : Type u_4) (𝓨 : Type u_5)
  [MeasurableSpace 𝓐]
  [MeasurableSpace 𝓨] : Type (max u_4 u_5)

Code

structure Algorithm (𝓐 𝓨 : Type*) [MeasurableSpace 𝓐] [MeasurableSpace 𝓨] where
  /-- Policy or sampling rule: distribution of the next action. -/
  policy : (n : ℕ) → Kernel (Iic n → 𝓐 × 𝓨) 𝓐
  /-- The policy is a Markov kernel. -/
  [h_policy : ∀ n, IsMarkovKernel (policy n)]
  /-- Distribution of the first action. -/
  p0 : Measure 𝓐
  /-- The first action distribution is a probability measure. -/
  [hp0 : IsProbabilityMeasure p0]

Used by (216)

Actions: Source · Open Issue

`probSpace`🔗

DefinitionBandits.ArrayModel.probSpace

Details

Probability space for the array model of stochastic bandits.

def

Bandits.ArrayModel.probSpace.{u_1, u_2} (𝓐 : Type u_1) (R : Type u_2) :
  Type (max u_1 u_2)
Bandits.ArrayModel.probSpace.{u_1, u_2}
  (𝓐 : Type u_1) (R : Type u_2) :
  Type (max u_1 u_2)

Code

def probSpace : Type _ := (ℕ → I) × (ℕ → 𝓐 → R)

Used by (64)

Actions: Source · Open Issue

`instMeasurableSpaceProbSpace`🔗

InstanceBandits.ArrayModel.instMeasurableSpaceProbSpace

Details

No docstring.

def

Bandits.ArrayModel.instMeasurableSpaceProbSpace.{u_3, u_4}
  {𝓐 : Type u_3} {R : Type u_4} [MeasurableSpace R] :
  MeasurableSpace (probSpace 𝓐 R)
Bandits.ArrayModel.instMeasurableSpaceProbSpace.{u_3,
    u_4}
  {𝓐 : Type u_3} {R : Type u_4}
  [MeasurableSpace R] :
  MeasurableSpace (probSpace 𝓐 R)

Code

instance {𝓐 R : Type*} [MeasurableSpace R] : MeasurableSpace (probSpace 𝓐 R)

Type uses (1)

probSpace

Used by (41)

Actions: Source · Open Issue

Proof

inferInstanceAs (MeasurableSpace ((ℕ → I) × (ℕ → 𝓐 → R)))

`reward`🔗

DefinitionBandits.ArrayModel.reward

Details

Reward received at time n in the array model.

def

Bandits.ArrayModel.reward.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐] (alg : Learning.Algorithm 𝓐 R)
  (n : ℕ) (ω : probSpace 𝓐 R) : R
Bandits.ArrayModel.reward.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐]
  (alg : Learning.Algorithm 𝓐 R) (n : ℕ)
  (ω : probSpace 𝓐 R) : R

Code

noncomputable
def reward [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) (ω : probSpace 𝓐 R) : R :=
  (hist alg ω n ⟨n, by simp⟩).2

Type uses (2)

Body uses (1)

hist

Used by (24)

Actions: Source · Open Issue

`hist`🔗

DefinitionBandits.ArrayModel.hist

Details

History of actions and rewards up to time n in the array model.

def

Bandits.ArrayModel.hist.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐] (alg : Learning.Algorithm 𝓐 R)
  (ω : probSpace 𝓐 R) (n : ℕ) : ↥(Finset.Iic n) → 𝓐 × R
Bandits.ArrayModel.hist.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐]
  (alg : Learning.Algorithm 𝓐 R)
  (ω : probSpace 𝓐 R) (n : ℕ) :
  ↥(Finset.Iic n) → 𝓐 × R

Code

noncomputable
def hist [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (ω : probSpace 𝓐 R) : (n : ℕ) → Iic n → 𝓐 × R
| 0 => fun _ ↦ (initAlgFunction alg (ω.1 0), ω.2 0 (initAlgFunction alg (ω.1 0)))
| n + 1 =>
  let hn : Iic n → 𝓐 × R := hist alg ω n
  let a : 𝓐 := algFunction alg n hn (ω.1 (n + 1))
  fun i ↦ if hin : i ≤ n then hn ⟨i, by simp [hin]⟩ else (a, ω.2 (pullCount' n hn a) a)

Type uses (2)

Body uses (3)

Used by (30)

Actions: Source · Open Issue

`action`🔗

DefinitionBandits.ArrayModel.action

Details

Action taken at time n in the array model.

def

Bandits.ArrayModel.action.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐] (alg : Learning.Algorithm 𝓐 R)
  (n : ℕ) (ω : probSpace 𝓐 R) : 𝓐
Bandits.ArrayModel.action.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] [DecidableEq 𝓐]
  (alg : Learning.Algorithm 𝓐 R) (n : ℕ)
  (ω : probSpace 𝓐 R) : 𝓐

Code

noncomputable
def action [DecidableEq 𝓐] (alg : Algorithm 𝓐 R) (n : ℕ) (ω : probSpace 𝓐 R) : 𝓐 :=
  (hist alg ω n ⟨n, by simp⟩).1

Type uses (2)

Body uses (1)

hist

Used by (43)

Actions: Source · Open Issue

`arrayMeasure`🔗

DefinitionBandits.ArrayModel.arrayMeasure

Details

Probability measure for the array model of stochastic bandits.

def

Bandits.ArrayModel.arrayMeasure.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R}
  (ν : ProbabilityTheory.Kernel 𝓐 R) :
  MeasureTheory.Measure (probSpace 𝓐 R)
Bandits.ArrayModel.arrayMeasure.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R}
  (ν : ProbabilityTheory.Kernel 𝓐 R) :
  MeasureTheory.Measure (probSpace 𝓐 R)

Code

noncomputable
def arrayMeasure (ν : Kernel 𝓐 R) : Measure (probSpace 𝓐 R) :=
  (Measure.infinitePi fun _ ↦ volume).prod (streamMeasure ν)

Type uses (2)

Body uses (1)

streamMeasure

Used by (29)

Actions: Source · Open Issue

`instIsProbabilityMeasureP0`🔗

InstanceLearning.instIsProbabilityMeasureP0

Details

No docstring.

theorem

Learning.instIsProbabilityMeasureP0.{u_1, u_2} {𝓐 : Type u_1}
  {𝓨 : Type u_2} {m𝓐 : MeasurableSpace 𝓐} {m𝓨 : MeasurableSpace 𝓨}
  (alg : Algorithm 𝓐 𝓨) :
  MeasureTheory.IsProbabilityMeasure (Algorithm.p0 alg)
Learning.instIsProbabilityMeasureP0.{u_1,
    u_2}
  {𝓐 : Type u_1} {𝓨 : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {m𝓨 : MeasurableSpace 𝓨}
  (alg : Algorithm 𝓐 𝓨) :
  MeasureTheory.IsProbabilityMeasure
    (Algorithm.p0 alg)

Code

instance (alg : Algorithm 𝓐 𝓨) : IsProbabilityMeasure alg.p0

Type uses (1)

Algorithm

Used by (13)

Actions: Source · Open Issue

Proof

alg.hp0

`initAlgFunction`🔗

DefinitionBandits.ArrayModel.initAlgFunction

Details

The initial action is the image of a uniform random variable by this function.

def

Bandits.ArrayModel.initAlgFunction.{u_1, u_2} {𝓐 : Type u_1}
  {R : Type u_2} {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R}
  [Nonempty 𝓐] [StandardBorelSpace 𝓐] (alg : Learning.Algorithm 𝓐 R) :
  ↑unitInterval → 𝓐
Bandits.ArrayModel.initAlgFunction.{u_1,
    u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐]
  (alg : Learning.Algorithm 𝓐 R) :
  ↑unitInterval → 𝓐

Code

noncomputable
def initAlgFunction (alg : Algorithm 𝓐 R) : I → 𝓐 :=
  (Measure.exists_measurable_map_eq alg.p0).choose

Type uses (1)

Algorithm

Body uses (1)

instIsProbabilityMeasureP0

Used by (12)

Actions: Source · Open Issue

`instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy`🔗

InstanceLearning.instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy

Details

No docstring.

theorem

Learning.instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy.{u_1,
    u_2}
  {𝓐 : Type u_1} {𝓨 : Type u_2} {m𝓐 : MeasurableSpace 𝓐}
  {m𝓨 : MeasurableSpace 𝓨} (alg : Algorithm 𝓐 𝓨) (n : ℕ) :
  ProbabilityTheory.IsMarkovKernel (Algorithm.policy alg n)
Learning.instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy.{u_1,
    u_2}
  {𝓐 : Type u_1} {𝓨 : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {m𝓨 : MeasurableSpace 𝓨}
  (alg : Algorithm 𝓐 𝓨) (n : ℕ) :
  ProbabilityTheory.IsMarkovKernel
    (Algorithm.policy alg n)

Code

instance (alg : Algorithm 𝓐 𝓨) (n : ℕ) : IsMarkovKernel (alg.policy n)

Type uses (1)

Algorithm

Used by (14)

Actions: Source · Open Issue

Proof

alg.h_policy n

`algFunction`🔗

DefinitionBandits.ArrayModel.algFunction

Details

The next action is the image of the history and a uniform random variable by this function.

def

Bandits.ArrayModel.algFunction.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐] (alg : Learning.Algorithm 𝓐 R) (n : ℕ) :
  (↥(Finset.Iic n) → 𝓐 × R) → ↑unitInterval → 𝓐
Bandits.ArrayModel.algFunction.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R} [Nonempty 𝓐]
  [StandardBorelSpace 𝓐]
  (alg : Learning.Algorithm 𝓐 R) (n : ℕ) :
  (↥(Finset.Iic n) → 𝓐 × R) →
    ↑unitInterval → 𝓐

Code

noncomputable
def algFunction (alg : Algorithm 𝓐 R) (n : ℕ) :
    (Iic n → 𝓐 × R) → I → 𝓐 :=
  (Kernel.exists_measurable_map_eq_unitInterval (alg.policy n)).choose

Type uses (1)

Algorithm

Body uses (1)

instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy

Used by (17)

Actions: Source · Open Issue

`pullCount'`🔗

DefinitionLearning.pullCount'

Details

Number of pulls of arm a up to (and including) time n. This is the number of entries in h in which the arm is a.

def

Learning.pullCount'.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  [DecidableEq 𝓐] (n : ℕ) (h : ↥(Finset.Iic n) → 𝓐 × R) (a : 𝓐) : ℕ
Learning.pullCount'.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  [DecidableEq 𝓐] (n : ℕ)
  (h : ↥(Finset.Iic n) → 𝓐 × R) (a : 𝓐) :
  ℕ

Code

noncomputable
def pullCount' (n : ℕ) (h : Iic n → 𝓐 × R) (a : 𝓐) := #{s | (h s).1 = a}

Used by (29)

Actions: Source · Open Issue

`streamMeasure`🔗

DefinitionBandits.streamMeasure

Details

Measure of an infinite stream of rewards from each action.

def

Bandits.streamMeasure.{u_1, u_2} {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐} {mR : MeasurableSpace R}
  (ν : ProbabilityTheory.Kernel 𝓐 R) : MeasureTheory.Measure (ℕ → 𝓐 → R)
Bandits.streamMeasure.{u_1, u_2}
  {𝓐 : Type u_1} {R : Type u_2}
  {m𝓐 : MeasurableSpace 𝓐}
  {mR : MeasurableSpace R}
  (ν : ProbabilityTheory.Kernel 𝓐 R) :
  MeasureTheory.Measure (ℕ → 𝓐 → R)

Code

noncomputable
def streamMeasure (ν : Kernel 𝓐 R) : Measure (ℕ → 𝓐 → R) :=
  Measure.infinitePi fun _ ↦ Measure.infinitePi ν

Used by (56)

Actions: Source · Open Issue

LeanMachineLearning exposition

`Bandits.ArrayModel.hasCondDistrib_reward'`🔗

`hasCondDistrib_reward'`🔗

`Algorithm`🔗

`probSpace`🔗

`instMeasurableSpaceProbSpace`🔗

`reward`🔗

`hist`🔗

`action`🔗

`arrayMeasure`🔗

`instIsProbabilityMeasureP0`🔗

`initAlgFunction`🔗

`instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy`🔗

`algFunction`🔗

`pullCount'`🔗

`streamMeasure`🔗

Bandits.ArrayModel.hasCondDistrib_reward'🔗

hasCondDistrib_reward'🔗

Algorithm🔗

probSpace🔗

instMeasurableSpaceProbSpace🔗

reward🔗

hist🔗

action🔗

arrayMeasure🔗

instIsProbabilityMeasureP0🔗

initAlgFunction🔗

instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy🔗

algFunction🔗

pullCount'🔗

streamMeasure🔗

`Bandits.ArrayModel.hasCondDistrib_reward'`🔗

`hasCondDistrib_reward'`🔗

`Algorithm`🔗

`probSpace`🔗

`instMeasurableSpaceProbSpace`🔗

`reward`🔗

`hist`🔗

`action`🔗

`arrayMeasure`🔗

`instIsProbabilityMeasureP0`🔗

`initAlgFunction`🔗

`instIsMarkovKernelForallSubtypeNatMemFinsetIicProdPolicy`🔗

`algFunction`🔗

`pullCount'`🔗

`streamMeasure`🔗