I tried these 3 solutions second being the winner. And the third is the same as second but using ALL keyword.
-- Solution 1
;WITH correctInfo AS (SELECT GivenPartNumber_non, vcompanyID FROM #notmappedsources
GROUP BY GivenPartNumber_Non, vcompanyid
HAVING MIN(sourceType) = 484456 AND MAX(SourceType) = 484456)
SELECT n.* FROM #notmappedsources n INNER JOIN correctInfo c
ON c.vcompanyid = n.vcompanyid AND n.GivenPartNumber_Non = c.givenPartNumber_non;
-------- Solution 2
SELECT n.* FROM #notmappedsources n WHERE NOT EXISTS (SELECT 1 FROM #notmappedsources n2 WHERE n2.GivenPartNumber_Non = n.GivenPartNumber_Non
AND n2.vcompanyid = n.vcompanyid AND n2.SourceType<>484456)
-------- Solution 3
SELECT n.* FROM #notmappedsources n WHERE 484456 = all (SELECT sourceType FROM #notmappedsources n2 WHERE n2.GivenPartNumber_Non = n.GivenPartNumber_Non
AND n2.vcompanyid = n.vcompanyid)