LeanMachineLearning exposition

Bandits.prob_sumRewards_sub_pullCount_mul_le_le๐Ÿ”—

This page has the declaration's own card below, then its dependency graph, then a card for each dependency (type dependencies first, then the rest of the transitive closure). For a theorem, the graph and the dependency cards only follow its statement's dependencies (its proof is replaced by sorry, so what it proves doesn't depend on how); for everything else, both the type and the body/value are followed, since their content is part of what later declarations build on.

Minimal Lean file

prob_sumRewards_sub_pullCount_mul_le_le๐Ÿ”—

LemmaBandits.prob_sumRewards_sub_pullCount_mul_le_le
Details

No docstring.

๐Ÿ”—theorem
Bandits.prob_sumRewards_sub_pullCount_mul_le_le.{u_1, u_2}
  {๐“ : Type u_1} {ฮฉ : Type u_2} [DecidableEq ๐“] {m๐“ : MeasurableSpace ๐“}
  {mฮฉ : MeasurableSpace ฮฉ} {P : MeasureTheory.Measure ฮฉ}
  [MeasureTheory.IsProbabilityMeasure P] {alg : Learning.Algorithm ๐“ โ„}
  {ฮฝ : ProbabilityTheory.Kernel ๐“ โ„}
  [ProbabilityTheory.IsMarkovKernel ฮฝ] {A : โ„• โ†’ ฮฉ โ†’ ๐“} {R : โ„• โ†’ ฮฉ โ†’ โ„}
  {n : โ„•} {a : ๐“} [Nonempty ๐“] [Countable ๐“]
  [MeasurableSingletonClass ๐“] {ฯƒ2 : NNReal} (hฯƒ2 : 0 < ฯƒ2)
  (ha :
    ProbabilityTheory.HasSubgaussianMGF
      (fun x => x - โˆซ (x : โ„), id x โˆ‚ฮฝ a) ฯƒ2 (ฮฝ a))
  (h : Learning.IsAlgEnvSeq A R alg (Learning.stationaryEnv ฮฝ) P)
  {ฮด : โ„} (hฮด : 0 < ฮด) :
  P
      {ฯ‰ |
        โˆƒ t < n,
          Learning.pullCount A a t ฯ‰ โ‰  0 โˆง
            Learning.sumRewards A R a t ฯ‰ -
                โ†‘(Learning.pullCount A a t ฯ‰) * โˆซ (x : โ„), id x โˆ‚ฮฝ a โ‰ค
              -โˆš(2 * โ†‘(Learning.pullCount A a t ฯ‰) * โ†‘ฯƒ2 *
                    Real.log (1 / ฮด))} โ‰ค
    ENNReal.ofReal ((โ†‘n - 1) * ฮด)
Bandits.prob_sumRewards_sub_pullCount_mul_le_le.{u_1,
    u_2}
  {๐“ : Type u_1} {ฮฉ : Type u_2}
  [DecidableEq ๐“] {m๐“ : MeasurableSpace ๐“}
  {mฮฉ : MeasurableSpace ฮฉ}
  {P : MeasureTheory.Measure ฮฉ}
  [MeasureTheory.IsProbabilityMeasure P]
  {alg : Learning.Algorithm ๐“ โ„}
  {ฮฝ : ProbabilityTheory.Kernel ๐“ โ„}
  [ProbabilityTheory.IsMarkovKernel ฮฝ]
  {A : โ„• โ†’ ฮฉ โ†’ ๐“} {R : โ„• โ†’ ฮฉ โ†’ โ„} {n : โ„•}
  {a : ๐“} [Nonempty ๐“] [Countable ๐“]
  [MeasurableSingletonClass ๐“]
  {ฯƒ2 : NNReal} (hฯƒ2 : 0 < ฯƒ2)
  (ha :
    ProbabilityTheory.HasSubgaussianMGF
      (fun x => x - โˆซ (x : โ„), id x โˆ‚ฮฝ a)
      ฯƒ2 (ฮฝ a))
  (h :
    Learning.IsAlgEnvSeq A R alg
      (Learning.stationaryEnv ฮฝ) P)
  {ฮด : โ„} (hฮด : 0 < ฮด) :
  P
      {ฯ‰ |
        โˆƒ t < n,
          Learning.pullCount A a t ฯ‰ โ‰  0 โˆง
            Learning.sumRewards A R a t
                  ฯ‰ -
                โ†‘(Learning.pullCount A a t
                      ฯ‰) *
                  โˆซ (x : โ„), id x โˆ‚ฮฝ a โ‰ค
              -โˆš(2 *
                        โ†‘(Learning.pullCount
                            A a t ฯ‰) *
                      โ†‘ฯƒ2 *
                    Real.log (1 / ฮด))} โ‰ค
    ENNReal.ofReal ((โ†‘n - 1) * ฮด)

Code

lemma prob_sumRewards_sub_pullCount_mul_le_le [Countable ๐“] [MeasurableSingletonClass ๐“]
    {ฯƒ2 : โ„โ‰ฅ0} (hฯƒ2 : 0 < ฯƒ2) (ha : HasSubgaussianMGF (fun x โ†ฆ x - (ฮฝ a)[id]) ฯƒ2 (ฮฝ a))
    (h : IsAlgEnvSeq A R alg (stationaryEnv ฮฝ) P) {ฮด : โ„} (hฮด : 0 < ฮด) :
    P {ฯ‰ | โˆƒ t < n, pullCount A a t ฯ‰ โ‰  0 โˆง
      sumRewards A R a t ฯ‰ - pullCount A a t ฯ‰ * (ฮฝ a)[id] โ‰ค
        -โˆš(2 * pullCount A a t ฯ‰ * ฯƒ2 * Real.log (1 / ฮด))} โ‰ค ENNReal.ofReal ((n - 1) * ฮด)
Type uses (5)
Body uses (4)
Used by (1)

Actions: Source ยท Open Issue

Proof
let B (m : โ„•) := {x : โ„ | x - m * (ฮฝ a)[id] โ‰ค -โˆš(2 * m * ฯƒ2 * Real.log (1 / ฮด))}
  calc
    _ โ‰ค P (โ‹ƒ m โˆˆ Icc 1 (n - 1), {ฯ‰ | โˆƒ t, t < n โˆง pullCount A a t ฯ‰ = m โˆง
            sumRewards A R a t ฯ‰ โˆˆ B m}) := by
        apply measure_mono
        intro ฯ‰ โŸจt, ht, hp, hbโŸฉ
        have hm : pullCount A a t ฯ‰ โˆˆ Icc 1 (n - 1) := mem_Icc.mpr โŸจNat.one_le_iff_ne_zero.mpr hp,
          (pullCount_le a t ฯ‰).trans (Nat.le_sub_one_of_lt ht)โŸฉ
        exact Set.mem_biUnion hm โŸจt, ht, rfl, hbโŸฉ
    _ โ‰ค โˆ‘ m โˆˆ Icc 1 (n - 1), P {ฯ‰ | โˆƒ t, t < n โˆง pullCount A a t ฯ‰ = m โˆง
          sumRewards A R a t ฯ‰ โˆˆ B m} :=
        measure_biUnion_finset_le _ _
    _ โ‰ค โˆ‘ m โˆˆ Icc 1 (n - 1), P {ฯ‰ | โˆƒ t, pullCount A a t ฯ‰ = m โˆง sumRewards A R a t ฯ‰ โˆˆ B m} :=
        sum_le_sum (fun _ _ โ†ฆ measure_mono (fun _ โŸจt, _, hpsโŸฉ โ†ฆ โŸจt, hpsโŸฉ))
    _ โ‰ค โˆ‘ m โˆˆ Icc 1 (n - 1), streamMeasure ฮฝ {ฯ‰ | โˆ‘ i โˆˆ range m, ฯ‰ i a โˆˆ B m} := by
        apply sum_le_sum
        exact (fun m _ โ†ฆ prob_exists_pullCount_eq_and_sumRewards_mem_le h a m (by measurability))
    _ โ‰ค โˆ‘ m โˆˆ Icc 1 (n - 1), ENNReal.ofReal ฮด := by
      apply sum_le_sum
      intro m hm
      exact le_of_eq_of_le (by simp [B])
        (StreamMeasure.prob_sum_range_sub_le_le_of_HasSubgaussianMGF' hฯƒ2 ha hฮด (mem_Icc.mp hm).1)
    _ = ENNReal.ofReal ((n - 1) * ฮด) := by
      by_cases hn : n = 0
      ยท simp [hn, hฮด.le]
      ยท rw [sum_const, Nat.card_Icc, add_tsub_cancel_right, โ† ENNReal.ofReal_nsmul, nsmul_eq_mul,
          Nat.cast_sub (Nat.one_le_iff_ne_zero.mpr hn)]
        ring_nf

Dependency graph

Type dependencies (5)

Algorithm๐Ÿ”—

StructureLearning.Algorithm
Details

A stochastic, sequential algorithm.

๐Ÿ”—structure
Learning.Algorithm.{u_4, u_5} (๐“ : Type u_4) (๐“จ : Type u_5)
  [MeasurableSpace ๐“] [MeasurableSpace ๐“จ] : Type (max u_4 u_5)
Learning.Algorithm.{u_4, u_5}
  (๐“ : Type u_4) (๐“จ : Type u_5)
  [MeasurableSpace ๐“]
  [MeasurableSpace ๐“จ] : Type (max u_4 u_5)

Code

structure Algorithm (๐“ ๐“จ : Type*) [MeasurableSpace ๐“] [MeasurableSpace ๐“จ] where
  /-- Policy or sampling rule: distribution of the next action. -/
  policy : (n : โ„•) โ†’ Kernel (Iic n โ†’ ๐“ ร— ๐“จ) ๐“
  /-- The policy is a Markov kernel. -/
  [h_policy : โˆ€ n, IsMarkovKernel (policy n)]
  /-- Distribution of the first action. -/
  p0 : Measure ๐“
  /-- The first action distribution is a probability measure. -/
  [hp0 : IsProbabilityMeasure p0]
Used by (216)

Actions: Source ยท Open Issue

IsAlgEnvSeq๐Ÿ”—

StructureLearning.IsAlgEnvSeq
Details

An algorithm-environment sequence: a sequence of actions and feedbacks generated by an algorithm interacting with an environment.

๐Ÿ”—structure
Learning.IsAlgEnvSeq.{u_1, u_2, u_3} {๐“ : Type u_1} {๐“จ : Type u_2}
  {ฮฉ : Type u_3} {m๐“ : MeasurableSpace ๐“} {m๐“จ : MeasurableSpace ๐“จ}
  {mฮฉ : MeasurableSpace ฮฉ} (A : โ„• โ†’ ฮฉ โ†’ ๐“) (Y : โ„• โ†’ ฮฉ โ†’ ๐“จ)
  (alg : Algorithm ๐“ ๐“จ) (env : Environment ๐“ ๐“จ)
  (P : MeasureTheory.Measure ฮฉ) [MeasureTheory.IsFiniteMeasure P] : Prop
Learning.IsAlgEnvSeq.{u_1, u_2, u_3}
  {๐“ : Type u_1} {๐“จ : Type u_2}
  {ฮฉ : Type u_3} {m๐“ : MeasurableSpace ๐“}
  {m๐“จ : MeasurableSpace ๐“จ}
  {mฮฉ : MeasurableSpace ฮฉ} (A : โ„• โ†’ ฮฉ โ†’ ๐“)
  (Y : โ„• โ†’ ฮฉ โ†’ ๐“จ) (alg : Algorithm ๐“ ๐“จ)
  (env : Environment ๐“ ๐“จ)
  (P : MeasureTheory.Measure ฮฉ)
  [MeasureTheory.IsFiniteMeasure P] : Prop

Code

structure IsAlgEnvSeq
    (A : โ„• โ†’ ฮฉ โ†’ ๐“) (Y : โ„• โ†’ ฮฉ โ†’ ๐“จ) (alg : Algorithm ๐“ ๐“จ) (env : Environment ๐“ ๐“จ)
    (P : Measure ฮฉ) [IsFiniteMeasure P] : Prop where
  /-- The action sequence is measurable. -/
  measurable_action n : Measurable (A n) := by fun_prop
  /-- The feedback sequence is measurable. -/
  measurable_feedback n : Measurable (Y n) := by fun_prop
  /-- The first action has the correct law. -/
  hasLaw_action_zero : HasLaw (fun ฯ‰ โ†ฆ (A 0 ฯ‰)) alg.p0 P
  /-- The first feedback has the correct conditional distribution. -/
  hasCondDistrib_feedback_zero : HasCondDistrib (Y 0) (A 0) env.ฮฝ0 P
  /-- The next action has the correct conditional distribution given the history. -/
  hasCondDistrib_action n :
    HasCondDistrib (A (n + 1)) (history A Y n) (alg.policy n) P
  /-- The next feedback has the correct conditional distribution given the history and
  next action. -/
  hasCondDistrib_feedback n :
    HasCondDistrib (Y (n + 1)) (fun ฯ‰ โ†ฆ (history A Y n ฯ‰, A (n + 1) ฯ‰))
      (env.feedback n) P
Type uses (3)
Used by (111)

Actions: Source ยท Open Issue

stationaryEnv๐Ÿ”—

DefinitionLearning.stationaryEnv
Details

A stationary environment, in which the distribution of the next feedback depends only on the last action.

๐Ÿ”—def
Learning.stationaryEnv.{u_1, u_2} {๐“ : Type u_1} {๐“จ : Type u_2}
  {m๐“ : MeasurableSpace ๐“} {m๐“จ : MeasurableSpace ๐“จ}
  (ฮฝ : ProbabilityTheory.Kernel ๐“ ๐“จ)
  [ProbabilityTheory.IsMarkovKernel ฮฝ] : Environment ๐“ ๐“จ
Learning.stationaryEnv.{u_1, u_2}
  {๐“ : Type u_1} {๐“จ : Type u_2}
  {m๐“ : MeasurableSpace ๐“}
  {m๐“จ : MeasurableSpace ๐“จ}
  (ฮฝ : ProbabilityTheory.Kernel ๐“ ๐“จ)
  [ProbabilityTheory.IsMarkovKernel ฮฝ] :
  Environment ๐“ ๐“จ

Code

def stationaryEnv (ฮฝ : Kernel ๐“ ๐“จ) [IsMarkovKernel ฮฝ] : Environment ๐“ ๐“จ := obliviousEnv fun _ โ†ฆ ฮฝ
Type uses (1)
Body uses (1)
Used by (81)

Actions: Source ยท Open Issue

pullCount๐Ÿ”—

DefinitionLearning.pullCount
Details

Number of times action a was chosen up to time t (excluding t).

๐Ÿ”—def
Learning.pullCount.{u_1, u_3} {๐“ : Type u_1} {ฮฉ : Type u_3}
  [DecidableEq ๐“] (A : โ„• โ†’ ฮฉ โ†’ ๐“) (a : ๐“) (t : โ„•) (ฯ‰ : ฮฉ) : โ„•
Learning.pullCount.{u_1, u_3}
  {๐“ : Type u_1} {ฮฉ : Type u_3}
  [DecidableEq ๐“] (A : โ„• โ†’ ฮฉ โ†’ ๐“) (a : ๐“)
  (t : โ„•) (ฯ‰ : ฮฉ) : โ„•

Code

noncomputable
def pullCount (A : โ„• โ†’ ฮฉ โ†’ ๐“) (a : ๐“) (t : โ„•) (ฯ‰ : ฮฉ) : โ„• :=
  #(filter (fun s โ†ฆ A s ฯ‰ = a) (range t))
Used by (146)

Actions: Source ยท Open Issue

sumRewards๐Ÿ”—

DefinitionLearning.sumRewards
Details

Sum of rewards obtained when pulling action a up to time t (exclusive).

๐Ÿ”—def
Learning.sumRewards.{u_1, u_3} {๐“ : Type u_1} {ฮฉ : Type u_3}
  [DecidableEq ๐“] (A : โ„• โ†’ ฮฉ โ†’ ๐“) (R' : โ„• โ†’ ฮฉ โ†’ โ„) (a : ๐“) (t : โ„•)
  (ฯ‰ : ฮฉ) : โ„
Learning.sumRewards.{u_1, u_3}
  {๐“ : Type u_1} {ฮฉ : Type u_3}
  [DecidableEq ๐“] (A : โ„• โ†’ ฮฉ โ†’ ๐“)
  (R' : โ„• โ†’ ฮฉ โ†’ โ„) (a : ๐“) (t : โ„•)
  (ฯ‰ : ฮฉ) : โ„

Code

def sumRewards (A : โ„• โ†’ ฮฉ โ†’ ๐“) (R' : โ„• โ†’ ฮฉ โ†’ โ„) (a : ๐“) (t : โ„•) (ฯ‰ : ฮฉ) : โ„ :=
  โˆ‘ s โˆˆ range t, if A s ฯ‰ = a then R' s ฯ‰ else 0
Used by (44)

Actions: Source ยท Open Issue

All dependencies, transitively (3)

Environment๐Ÿ”—

StructureLearning.Environment
Details

A stochastic environment.

๐Ÿ”—structure
Learning.Environment.{u_4, u_5} (๐“ : Type u_4) (๐“จ : Type u_5)
  [MeasurableSpace ๐“] [MeasurableSpace ๐“จ] : Type (max u_4 u_5)
Learning.Environment.{u_4, u_5}
  (๐“ : Type u_4) (๐“จ : Type u_5)
  [MeasurableSpace ๐“]
  [MeasurableSpace ๐“จ] : Type (max u_4 u_5)

Code

structure Environment (๐“ ๐“จ : Type*) [MeasurableSpace ๐“] [MeasurableSpace ๐“จ] where
  /-- Distribution of the next observation as function of the past history. -/
  feedback : (n : โ„•) โ†’ Kernel ((Iic n โ†’ ๐“ ร— ๐“จ) ร— ๐“) ๐“จ
  /-- The feedback kernels are Markov kernels. -/
  [h_feedback : โˆ€ n, IsMarkovKernel (feedback n)]
  /-- Distribution of the first observation given the first action. -/
  ฮฝ0 : Kernel ๐“ ๐“จ
  /-- The initial observation kernel is a Markov kernel. -/
  [hp0 : IsMarkovKernel ฮฝ0]
Used by (128)

Actions: Source ยท Open Issue

history๐Ÿ”—

DefinitionLearning.history
Details

History of the algorithm-environment sequence up to time n.

๐Ÿ”—def
Learning.history.{u_1, u_2, u_3} {๐“ : Type u_1} {๐“จ : Type u_2}
  {ฮฉ : Type u_3} (A : โ„• โ†’ ฮฉ โ†’ ๐“) (Y : โ„• โ†’ ฮฉ โ†’ ๐“จ) (n : โ„•) (ฯ‰ : ฮฉ) :
  โ†ฅ(Finset.Iic n) โ†’ ๐“ ร— ๐“จ
Learning.history.{u_1, u_2, u_3}
  {๐“ : Type u_1} {๐“จ : Type u_2}
  {ฮฉ : Type u_3} (A : โ„• โ†’ ฮฉ โ†’ ๐“)
  (Y : โ„• โ†’ ฮฉ โ†’ ๐“จ) (n : โ„•) (ฯ‰ : ฮฉ) :
  โ†ฅ(Finset.Iic n) โ†’ ๐“ ร— ๐“จ

Code

def history (A : โ„• โ†’ ฮฉ โ†’ ๐“) (Y : โ„• โ†’ ฮฉ โ†’ ๐“จ) (n : โ„•) (ฯ‰ : ฮฉ) : Iic n โ†’ ๐“ ร— ๐“จ :=
  fun i โ†ฆ (A i ฯ‰, Y i ฯ‰)
Used by (72)

Actions: Source ยท Open Issue

obliviousEnv๐Ÿ”—

DefinitionLearning.obliviousEnv
Details

An oblivious environment, in which the distribution of the next feedback depends only on the last action, but in a possibly time-dependent manner.

๐Ÿ”—def
Learning.obliviousEnv.{u_1, u_2} {๐“ : Type u_1} {๐“จ : Type u_2}
  {m๐“ : MeasurableSpace ๐“} {m๐“จ : MeasurableSpace ๐“จ}
  (ฮฝ : โ„• โ†’ ProbabilityTheory.Kernel ๐“ ๐“จ)
  [โˆ€ (n : โ„•), ProbabilityTheory.IsMarkovKernel (ฮฝ n)] : Environment ๐“ ๐“จ
Learning.obliviousEnv.{u_1, u_2}
  {๐“ : Type u_1} {๐“จ : Type u_2}
  {m๐“ : MeasurableSpace ๐“}
  {m๐“จ : MeasurableSpace ๐“จ}
  (ฮฝ : โ„• โ†’ ProbabilityTheory.Kernel ๐“ ๐“จ)
  [โˆ€ (n : โ„•),
      ProbabilityTheory.IsMarkovKernel
        (ฮฝ n)] :
  Environment ๐“ ๐“จ

Code

def obliviousEnv (ฮฝ : โ„• โ†’ Kernel ๐“ ๐“จ) [โˆ€ n, IsMarkovKernel (ฮฝ n)] : Environment ๐“ ๐“จ where
  feedback n := (ฮฝ (n + 1)).prodMkLeft _
  ฮฝ0 := ฮฝ 0
Type uses (1)
Used by (10)

Actions: Source ยท Open Issue