Grab and Drop Script goes through walls.

Hey, I've tried a lot of things to get this working, but no luck. I'm using standard character controller. When I pickup an object, it goes through walls and floors. My object has a sphere collider, it isn't trigger. I've tried setting kinematic and setting parent. I added Layers but that didn't do anything either. Any ideas? I tried the DragRigidbody script but couldn't get that working either :(

Thanks in advance.

  using UnityEngine;
  using System.Collections;

 public class GrabAndDrop : MonoBehaviour {
       GameObject grabbedObject;
       float grabbedObjectSize;
       public Transform Player;
     
 GameObject GetMouseHoverObject(float range)
 {
     Vector3 position = gameObject.transform.position;
     RaycastHit raycastHit;
     Vector3 target = position + Camera.main.transform.forward * range;

     if (Physics.Linecast(position,target,out raycastHit))
         return raycastHit.collider.gameObject;
     return null;
     
 }

 bool CanGrab(GameObject candidate)
 {
     return candidate.GetComponent<Rigidbody>() != null;
 }

 void TryGrabObject(GameObject grabObject)
 {
     if (grabObject == null || !CanGrab(grabObject))
         return;

     grabbedObject = grabObject;
     grabbedObjectSize = grabObject.GetComponent<Renderer>().bounds.size.magnitude;
     //grabbedObject.GetComponent<SphereCollider>().enabled = false;
     //grabbedObject.GetComponent<Rigidbody>().isKinematic = true;
     //grabbedObject.transform.SetParent(Player);
 }

 void DropObject ()
 {
     if (grabbedObject == null)
         return;
     if (grabbedObject.GetComponent<Rigidbody>() != null)
         grabbedObject.GetComponent<Rigidbody>().velocity = Vector3.zero;
     //grabbedObject.GetComponent<SphereCollider>().enabled = true;
     //grabbedObject.GetComponent<Rigidbody>().isKinematic = false;
     //grabbedObject.transform.SetParent(null);


     grabbedObject = null;
 }
 void Update () {
     if (Input.GetKeyDown(KeyCode.E))
     {
         if (grabbedObject == null)
             TryGrabObject(GetMouseHoverObject(10));
         else
             DropObject();
     
     }
     if (grabbedObject != null)
     {
         Vector3 newPosition = gameObject.transform.position + Camera.main.transform.forward * grabbedObjectSize;
         grabbedObject.transform.position = newPosition;
         grabbedObject.transform.rotation = Quaternion.identity;
     }
 }

}