Check a query:
select *,
case when m % 3 = 0 then
isnull((select ReturnValue from ATable where year(EffectiveDate) = y and month(EffectiveDate) = m - 2), 1) *
isnull((select ReturnValue from ATable where year(EffectiveDate) = y and month(EffectiveDate) = m - 1), 1) *
ReturnValue
else ReturnValue end as NewColumn
from ATable
cross apply (values (year(EffectiveDate), month(EffectiveDate))) M(y, m)